Fix missing dags ui

.coveragerc

@@ -23,3 +23,4 @@ omit =
     scripts/*
     dev/*
     airflow/migrations/*
+    airflow/www_rbac/node_modules/**

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 110
+ignore = E731,W504
+exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*/_vendor/*

.github/PULL_REQUEST_TEMPLATE.md

@@ -30,4 +30,4 @@ Make sure you have checked _all_ steps below.
 
 
 ### Code Quality
-- [ ] Passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
+- [ ] Passes `flake8`

.gitignore

@@ -83,6 +83,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/_api/
 
 # PyBuilder
 target/
@@ -138,3 +139,22 @@ rat-results.txt
 *.generated
 *.tar.gz
 scripts/ci/kubernetes/kube/.generated/airflow.yaml
+
+# Node & Webpack Stuff
+*.entry.js
+node_modules
+npm-debug.log*
+static/dist
+derby.log
+metastore_db
+
+# Airflow log files when airflow is run locally
+airflow-*.err
+airflow-*.out
+airflow-*.log
+airflow-*.pid
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json

.rat-excludes

@@ -1,33 +1,45 @@
+# Note: these patterns are applied to single files or directories, not full paths
+# coverage/* will ignore any coverage dir, but airflow/www/static/coverage/* will match nothing
+
 .gitignore
 .gitattributes
+.airflowignore
 .coverage
 .coveragerc
 .codecov.yml
+.eslintrc
+.eslintignore
+.flake8
 .rat-excludes
 requirements.txt
 .*log
 .travis.yml
 .*pyc
 .*lock
-docs
-.*md
 dist
 build
 airflow.egg-info
 apache_airflow.egg-info
 .idea
 metastore_db
 .*sql
+.*svg
 .*csv
 CHANGELOG.txt
 .*zip
 .*lock
+# Generated doc files
+.*html
+_build/*
+_static/*
+.buildinfo
+searchindex.js
+
 # Apache Rat does not detect BSD-2 clause properly
 # it is compatible according to http://www.apache.org/legal/resolved.html#category-a
 kerberos_auth.py
 airflow_api_auth_backend_kerberos_auth_py.html
 licenses/*
-airflow/www/static/docs
 parallel.js
 underscore.js
 jquery.dataTables.min.js
@@ -37,3 +49,16 @@ bootstrap-toggle.min.js
 bootstrap-toggle.min.css
 d3.v3.min.js
 ace.js
+node_modules/*
+.*json
+coverage/*
+git_version
+flake8_diff.sh
+coverage.xml
+
+rat-results.txt
+apache-airflow-.*\+source.tar.gz.*
+apache-airflow-.*\+bin.tar.gz.*
+
+# vendored modules
+_vendor/*

.readthedocs.yml

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,4 +22,4 @@ python:
         - doc
         - docker
         - gcp_api
-        - emr 
+        - emr

.travis.yml

@@ -16,96 +16,73 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-sudo: true
-dist: trusty
+dist: xenial
 language: python
-jdk:
-  - oraclejdk8
-services:
-  - cassandra
-  - mysql
-  - postgresql
-  - rabbitmq
-addons:
-  apt:
-    packages:
-      - slapd
-      - ldap-utils
-      - openssh-server
-      - mysql-server-5.6
-      - mysql-client-core-5.6
-      - mysql-client-5.6
-      - krb5-user
-      - krb5-kdc
-      - krb5-admin-server
-      - oracle-java8-installer
-      - python-selinux
-  postgresql: "9.2"
 python:
-  - "2.7"
-  - "3.5"
+  - "3.6"
 env:
   global:
     - TRAVIS_CACHE=$HOME/.travis_cache/
-    - KRB5_CONFIG=/etc/krb5.conf
-    - KRB5_KTNAME=/etc/airflow.keytab
-    # Travis on google cloud engine has a global /etc/boto.cfg that
-    # does not work with python 3
-    - BOTO_CONFIG=/tmp/bogusvalue
   matrix:
-    - TOX_ENV=py27-backend_mysql
-    - TOX_ENV=py27-backend_sqlite
-    - TOX_ENV=py27-backend_postgres
-    - TOX_ENV=py35-backend_mysql
-    - TOX_ENV=py35-backend_sqlite
-    - TOX_ENV=py35-backend_postgres
-    - TOX_ENV=flake8
-    - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
-    - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0
-matrix:
-  exclude:
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_mysql
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_sqlite
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_postgres
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_mysql
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_sqlite
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_postgres
-    - python: "2.7"
-      env: TOX_ENV=flake8
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0
+    - TOX_ENV=py27-backend_mysql-env_docker
+    - TOX_ENV=py27-backend_sqlite-env_docker
+    - TOX_ENV=py27-backend_postgres-env_docker
+    - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py27-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.9.0
+    - TOX_ENV=py35-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.13.0 PYTHON_VERSION=3
+
+stages:
+  - pre-test
+  - test
+
+jobs:
+  include:
+    - name: Flake8
+      stage: pre-test
+      install: pip install flake8
+      script: flake8
+    - name: mypy
+      stage: pre-test
+      install: pip install mypy
+      script: mypy airflow tests
+    - name: Check license header
+      stage: pre-test
+      install: skip
+      script: scripts/ci/6-check-license.sh
+    - name: Check docs
+      stage: pre-test
+      install: pip install -e .[doc]
+      script: docs/build.sh
+
 cache:
   directories:
     - $HOME/.wheelhouse/
+    - $HOME/.cache/pip
     - $HOME/.travis_cache/
 before_install:
-  - yes | ssh-keygen -t rsa -C your_email@youremail.com -P '' -f ~/.ssh/id_rsa
-  - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-  - ln -s ~/.ssh/authorized_keys ~/.ssh/authorized_keys2
-  - chmod 600 ~/.ssh/*
-  - jdk_switcher use oraclejdk8
-install:
-  - pip install --upgrade pip
-  - pip install tox
-  - pip install codecov
-before_script:
-  - cat "$TRAVIS_BUILD_DIR/scripts/ci/my.cnf" | sudo tee -a /etc/mysql/my.cnf
-  - mysql -e 'drop database if exists airflow; create database airflow' -u root
-  - sudo service mysql restart
-  - psql -c 'create database airflow;' -U postgres
-  - export PATH=${PATH}:/tmp/hive/bin
   # Required for K8s v1.10.x. See
   # https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783
-  - sudo mount --make-shared / && sudo service docker restart
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then sudo mount --make-shared / && sudo service docker restart; fi
+install:
+  - pip install --upgrade pip
+  - docker-compose -f scripts/ci/docker-compose.yml pull --quiet --parallel
 script:
-  - ./scripts/ci/travis_script.sh
-after_success:
-  - codecov
+  - if [ -z "$KUBERNETES_VERSION" ]; then
+      docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then
+      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
+      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
+      ./scripts/ci/kubernetes/kube/deploy.sh -d persistent_mode &&
+      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then
+      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
+      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
+      ./scripts/ci/kubernetes/kube/deploy.sh -d git_mode &&
+      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+before_cache:
+  - sudo chown -R travis:travis $HOME/.cache/pip $HOME/.wheelhouse/