Distribute dependencies

This change moves the dependencies required for Kubernetes stack to
stack itself, making sure we deal with the dependencies required for
it and no other stack.

This is achieved by
- moving operating system dependencies from engine core bindep.txt into
stack bindep.txt
- moving python dependencies from engine core requirements.txt into
stack requirements.txt
- taking setup-stack.sh script in use so engine core executes this script
from the stack

The benefits of this approach
- ability to control dependencies directly within the stack without
impacting other stacks and engine core
- package dependencies required for the stack itself and not everything
- allow identifying engine core dependencies so they can be made part of
stack packaging, enabling stacks to stop using leftovers of an earlier
deployment done for a different stack
- pave the way for multi distro support since some stacks may support
more than one distro and that support can be introduced and maintained
within corresponding stack
- deferred processing of dependencies allow to engine become functioning
faster than how it is know. also, the overall deployment time will be
shortened a bit with the help of this due to dealing with dependencies
based on stack

Depends-On: I713d0e333bc4a01c92e6aa6ae1a4e6f3d9efe2ad
Change-Id: I9508275fe581cea15af3c9e7dfa0b3342370460b

diff --git a/bindep.txt b/bindep.txt
new file mode 100644 (file)
index 0000000..5aee147
--- /dev/null
+++ b/bindep.txt
@@ -0,0 +1,21 @@
+#-------------------------------------------------------------------------------
+# This file contains operating system package requirements for Kubernetes stack.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.
+#-------------------------------------------------------------------------------
+
+# NOTE (fdegir): Kubernetes stack operating system package requirements
+# Kubernetes stack can be deployed on instances created using libvirt thus why
+# these packages are installed since we do not know what the target environment
+# is in advance, without doing further processing of user input.
+libvirt-bin     [platform:dpkg]
+libvirt-dev     [platform:dpkg]
+pkg-config      [platform:dpkg]
+python3-pymysql [platform:dpkg]
+python3-zmq     [platform:dpkg]
+qemu-utils      [platform:dpkg]
+qemu-kvm        [platform:dpkg]
+qemu-system-x86 [platform:dpkg]
+sgabios         [platform:dpkg]
+
+# vim: set ts=2 sw=2 expandtab:

diff --git a/playbooks/roles/package/tasks/pip.yaml b/playbooks/roles/package/tasks/pip.yaml
index a4a4642546926138eb61e0d7fe92396de18dd29c..d403ea7f666c9b6ddc2175ec2bbe2d1e9942af96 100644 (file)
--- a/playbooks/roles/package/tasks/pip.yaml
+++ b/playbooks/roles/package/tasks/pip.yaml
@@ -25,12 +25,20 @@
     - absent
     - directory
 
     - absent
     - directory
 

-- name: Download pip packages using requirements.txt file
+# NOTE (fdegir): This could perhaps be moved to a different playbook or to engine core
+# itself so the collection of core packages are common across stacks
+- name: Download engine core pip packages using requirements.txt file

   command: "pip download -r {{ engine_path }}/requirements.txt --no-cache"
   changed_when: false
   args:
     chdir: "{{ pip_folder }}"
 
   command: "pip download -r {{ engine_path }}/requirements.txt --no-cache"
   changed_when: false
   args:
     chdir: "{{ pip_folder }}"
 

+- name: Download stack pip packages using requirements.txt file
+  command: "pip download -r {{ engine_path }}/engine/stack/{{ stack_type }}/requirements.txt --no-cache"
+  changed_when: false
+  args:
+    chdir: "{{ pip_folder }}"
+

 - name: Copy pip.conf
   template:
     src: pip.conf.j2
 - name: Copy pip.conf
   template:
     src: pip.conf.j2

diff --git a/requirements.txt b/requirements.txt
new file mode 100644 (file)
index 0000000..9c766aa
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,221 @@
+#-------------------------------------------------------------------------------
+# This file contains python requirement version pins for Kubernetes stack.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.
+#-------------------------------------------------------------------------------
+
+# NOTE (fdegir): Kubernetes stack python requirements
+# Kubernetes stack can be deployed on instances provisioned from OpenStack, VMs
+# created using libvirt, or on baremetal nodes. The requirements listed below
+# contain all the python packages necessary for Kubernetes stack.
+alembic==1.4.2
+amqp==2.5.2
+asn1crypto==1.3.0
+attrs==19.3.0
+automaton==2.0.0
+Babel==2.8.0
+bcrypt==3.1.7
+beautifulsoup4==4.8.2
+cachetools==4.0.0
+certifi==2019.11.28
+chardet==3.0.4
+click==7.1.1
+cliff==3.0.0
+cmd2==0.8.9
+configparser==5.0.0
+construct==2.8.22
+debtcollector==2.0.0
+decorator==4.4.2
+dictdiffer==0.8.1
+diskimage-builder==2.34.1
+Django==2.2.11
+django-cors-headers==3.2.1
+django-filter==2.2.0
+django-health-check==3.12.1
+djangorestframework==3.11.0
+dnspython==1.15.0
+docker==4.2.0
+dogpile.cache==0.9.0
+dulwich==0.19.15
+dynaconf==2.2.3
+entrypoints==0.3
+eventlet==0.25.1
+extras==1.0.0
+fasteners==0.14.1
+fixtures==3.0.0
+flake8==3.7.9
+Flask==1.1.1
+Flask-Migrate==2.5.3
+Flask-Script==2.0.6
+Flask-SQLAlchemy==2.4.1
+Frozen-Flask==0.15
+futurist==2.1.0
+git-review==1.28.0
+gitdb==4.0.2
+gitdb2==4.0.2
+GitPython==3.1.0
+google-auth==1.12.0
+greenlet==0.4.15
+hacking==2.0.0
+idna==2.9
+ifaddr==0.1.6
+ironic==14.0.0
+ironic-inspector==10.0.0
+ironic-lib==4.1.0
+ironic-python-agent-builder==1.1.0
+iso8601==0.1.12
+itsdangerous==1.1.0
+jmespath==0.9.5
+jsonpatch==1.25
+jsonpath-rw==1.4.0
+jsonpointer==2.0
+jsonschema==3.2.0
+junit-xml==1.9
+keystoneauth1==3.18.0
+keystonemiddleware==8.0.0
+kolla==9.0.1
+kolla-ansible==9.0.1
+kombu==4.6.8
+kubernetes==11.0.0
+libvirt-python==6.1.0
+linecache2==1.0.0
+logutils==0.3.5
+lxml==4.5.0
+Mako==1.1.2
+mccabe==0.6.1
+monotonic==1.5
+more-itertools==8.2.0
+msgpack==0.6.2
+munch==2.5.0
+netaddr==0.7.19
+netifaces==0.10.9
+networkx==2.4
+oauthlib==3.1.0
+openshift==0.11.0
+openstacksdk==0.43.0
+os-client-config==2.0.0
+os-service-types==1.7.0
+os-traits==2.2.0
+osc-lib==2.0.0
+oslo.cache==2.2.0
+oslo.concurrency==4.0.1
+oslo.config==8.0.1
+oslo.context==3.0.0
+oslo.db==8.0.0
+oslo.i18n==4.0.0
+oslo.log==4.1.0
+oslo.messaging==12.0.0
+oslo.middleware==4.0.1
+oslo.policy==3.0.2
+oslo.reports==2.0.0
+oslo.rootwrap==6.0.1
+oslo.serialization==3.1.0
+oslo.service==2.1.0
+oslo.upgradecheck==1.0.0
+oslo.utils==4.1.0
+oslo.versionedobjects==2.0.1
+osprofiler==3.0.0
+paramiko==2.7.1
+Paste==3.4.0
+PasteDeploy==2.1.0
+pecan==1.3.3
+ply==3.11
+prettytable==0.7.2
+proliantutils==2.9.4
+psutil==5.7.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycadf==3.0.0
+pycodestyle==2.5.0
+pycryptodomex==3.9.7
+pyfakefs==4.0.2
+pyflakes==2.1.1
+pyghmi==1.5.13
+Pygments==2.6.1
+pyinotify==0.9.6
+PyMySQL==0.9.3
+PyNaCl==1.3.0
+pyOpenSSL==19.1.0
+pyparsing==2.4.6
+pyperclip==1.7.0
+pyrsistent==0.16.0
+pysendfile==2.0.1
+pysmi==0.3.4
+pysnmp==4.4.12
+python-box==3.4.6
+python-cinderclient==6.0.0
+python-dateutil==2.8.1
+python-dotenv==0.12.0
+python-editor==1.0.4
+python-glanceclient==3.0.0
+python-heatclient==2.0.0
+python-ironic-inspector-client==4.0.0
+python-ironicclient==4.0.0
+python-keystoneclient==3.22.0
+python-mimeparse==1.6.0
+python-neutronclient==7.1.0
+python-novaclient==16.0.0
+python-openstackclient==5.1.0
+python-string-utils==1.0.0
+python-subunit==1.4.0
+python-swiftclient==3.9.0
+pytz==2019.3
+pyzmq==19.0.0
+reno==3.0.0
+repoze.lru==0.7
+requests==2.23.0
+requests-oauthlib==1.3.0
+requestsexceptions==1.4.0
+retrying==1.3.3
+rfc3986==1.3.2
+Routes==2.4.1
+rsa==4.0
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
+shade==1.33.0
+simplegeneric==0.8.1
+simplejson==3.17.0
+smmap==3.0.1
+smmap2==3.0.1
+soupsieve==2.0
+SQLAlchemy==1.3.15
+sqlalchemy-migrate==0.13.0
+sqlparse==0.3.1
+statsd==3.3.0
+stevedore==1.32.0
+sushy==3.1.0
+Tempita==0.5.2
+tenacity==6.1.0
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.4.0
+toml==0.10.0
+tooz==2.2.0
+traceback2==1.4.0
+tzlocal==2.0.0
+unittest2==1.1.0
+urllib3==1.25.8
+vine==1.3.0
+virtualbmc==2.0.0
+virtualenv==20.0.14
+voluptuous==0.11.7
+waitress==1.4.3
+warlock==1.3.3
+wcwidth==0.1.9
+WebOb==1.8.6
+websocket-client==0.57.0
+WebTest==2.0.34
+Werkzeug==1.0.0
+whitenoise==5.0.1
+wrapt==1.12.1
+WSME==0.9.3
+XStatic==1.0.2
+XStatic-Bootstrap-SCSS==3.3.7.1
+XStatic-DataTables==1.10.15.1
+XStatic-jQuery==1.12.4.1
+XStatic-Patternfly==3.21.0.1
+XStatic-Patternfly-Bootstrap-Treeview==2.1.3.2
+yappi==1.2.3
+zeroconf==0.24.5
+
+# vim: set ts=2 sw=2 expandtab:

diff --git a/setup-stack.sh b/setup-stack.sh
index 3fde7ee7630deb44e75ee1307ae9a5c63743f0d7..d927afb1aeb7a98580c7ae2e97fba3ed33065640 100755 (executable)
--- a/setup-stack.sh
+++ b/setup-stack.sh
@@ -33,4 +33,21 @@ export STACK_ROOT_DIR
 export DEPLOY_SCENARIO="${DEPLOY_SCENARIO:=k8-multus-plugins}"
 echo "Info  : Deploy scenario is $DEPLOY_SCENARIO"
 
 export DEPLOY_SCENARIO="${DEPLOY_SCENARIO:=k8-multus-plugins}"
 echo "Info  : Deploy scenario is $DEPLOY_SCENARIO"
 

+# NOTE (fdegir): we need to install stack specific operating system requirements
+# and python packages before proceeding further
+echo "Info  : Install stack operating system requirements listed in stack bindep.txt using $PKG_MGR"
+cd "$STACK_ROOT_DIR"
+# bindep -b exits with non-zero if it identifies a missing package so we disable pipefail
+set +o pipefail
+# shellcheck disable=SC2046 disable=SC2086
+bindep -b &> /dev/null || redirect_cmd ${INSTALLER_CMD} $(bindep -b)
+set -o pipefail
+
+# TODO (fdegir): installation of pip packages fail due to not being able to find setuptools
+# as build dependency so we do --no-use-pep517 as workaround.
+# This needs to be revisited later on.
+echo "Info  : Install stack python packages listed in stack requirements.txt using pip"
+echo "        Please note that this may take some time"
+redirect_cmd pip install --no-use-pep517 --force-reinstall -r requirements.txt
+

 # vim: set ts=2 sw=2 expandtab:
 # vim: set ts=2 sw=2 expandtab:

diff --git a/test-requirements.txt b/test-requirements.txt
index 13213c7730ff05e9996910eec615a9d1ca276c6a..19b0cbd05a2b1e378605202c0e8211b9d6347961 100644 (file)
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,10 +1,56 @@
 #-------------------------------------------------------------------------------
 #-------------------------------------------------------------------------------

-# This file contains engine python requirement version pins.
-# Changing versions might have side effects!
+# This file contains python requirement version pins for tox.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.

 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 

-# NOTE: engine test requirements
-tox==3.14.3
+# NOTE (fdegir): tox python requirements
+alabaster==0.7.12
+ansible==2.9.6

 ansible-lint==4.1.0
 ansible-lint==4.1.0

-yamllint==1.19.0
+appdirs==1.4.4
+Babel==2.8.0
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+cryptography==3.1.1
+distlib==0.3.1
+docutils==0.16
+filelock==3.0.12
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==1.7.0
+importlib-resources==3.0.0
+Jinja2==2.11.2
+MarkupSafe==1.1.1
+packaging==20.4
+pathspec==0.8.0
+pluggy==0.13.1
+py==1.9.0
+pycparser==2.20
+Pygments==2.7.1
+pyparsing==2.4.7
+pytz==2020.1
+PyYAML==5.3.1
+requests==2.24.0
+rtcat-sphinx-theme==0.1.1
+ruamel.yaml==0.16.12
+ruamel.yaml.clib==0.2.2

 shellcheck-py==0.7.0.1
 shellcheck-py==0.7.0.1

+six==1.15.0
+snowballstemmer==2.0.0
+Sphinx==2.3.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+toml==0.10.1
+tox==3.14.3
+urllib3==1.25.10
+virtualenv==20.0.33
+yamllint==1.19.0
+zipp==3.3.0
+
+# vim: set ts=2 sw=2 expandtab: