Distribute dependencies 37/6237/2
authorFatih Degirmenci <fatih.degirmenci@est.tech>
Mon, 5 Oct 2020 09:36:12 +0000 (09:36 +0000)
committerFatih Degirmenci <fdegir@gmail.com>
Sun, 11 Oct 2020 10:54:01 +0000 (10:54 +0000)
This change moves the dependencies required for Kubernetes stack to
stack itself, making sure we deal with the dependencies required for
it and no other stack.

This is achieved by
- moving operating system dependencies from engine core bindep.txt into
stack bindep.txt
- moving python dependencies from engine core requirements.txt into
stack requirements.txt
- taking setup-stack.sh script in use so engine core executes this script
from the stack

The benefits of this approach
- ability to control dependencies directly within the stack without
impacting other stacks and engine core
- package dependencies required for the stack itself and not everything
- allow identifying engine core dependencies so they can be made part of
stack packaging, enabling stacks to stop using leftovers of an earlier
deployment done for a different stack
- pave the way for multi distro support since some stacks may support
more than one distro and that support can be introduced and maintained
within corresponding stack
- deferred processing of dependencies allow to engine become functioning
faster than how it is know. also, the overall deployment time will be
shortened a bit with the help of this due to dealing with dependencies
based on stack

Depends-On: I713d0e333bc4a01c92e6aa6ae1a4e6f3d9efe2ad
Change-Id: I9508275fe581cea15af3c9e7dfa0b3342370460b

bindep.txt [new file with mode: 0644]
playbooks/roles/package/tasks/pip.yaml
requirements.txt [new file with mode: 0644]
setup-stack.sh
test-requirements.txt

diff --git a/bindep.txt b/bindep.txt
new file mode 100644 (file)
index 0000000..5aee147
--- /dev/null
@@ -0,0 +1,21 @@
+#-------------------------------------------------------------------------------
+# This file contains operating system package requirements for Kubernetes stack.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.
+#-------------------------------------------------------------------------------
+
+# NOTE (fdegir): Kubernetes stack operating system package requirements
+# Kubernetes stack can be deployed on instances created using libvirt thus why
+# these packages are installed since we do not know what the target environment
+# is in advance, without doing further processing of user input.
+libvirt-bin     [platform:dpkg]
+libvirt-dev     [platform:dpkg]
+pkg-config      [platform:dpkg]
+python3-pymysql [platform:dpkg]
+python3-zmq     [platform:dpkg]
+qemu-utils      [platform:dpkg]
+qemu-kvm        [platform:dpkg]
+qemu-system-x86 [platform:dpkg]
+sgabios         [platform:dpkg]
+
+# vim: set ts=2 sw=2 expandtab:
index a4a4642..d403ea7 100644 (file)
     - absent
     - directory
 
-- name: Download pip packages using requirements.txt file
+# NOTE (fdegir): This could perhaps be moved to a different playbook or to engine core
+# itself so the collection of core packages are common across stacks
+- name: Download engine core pip packages using requirements.txt file
   command: "pip download -r {{ engine_path }}/requirements.txt --no-cache"
   changed_when: false
   args:
     chdir: "{{ pip_folder }}"
 
+- name: Download stack pip packages using requirements.txt file
+  command: "pip download -r {{ engine_path }}/engine/stack/{{ stack_type }}/requirements.txt --no-cache"
+  changed_when: false
+  args:
+    chdir: "{{ pip_folder }}"
+
 - name: Copy pip.conf
   template:
     src: pip.conf.j2
diff --git a/requirements.txt b/requirements.txt
new file mode 100644 (file)
index 0000000..9c766aa
--- /dev/null
@@ -0,0 +1,221 @@
+#-------------------------------------------------------------------------------
+# This file contains python requirement version pins for Kubernetes stack.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.
+#-------------------------------------------------------------------------------
+
+# NOTE (fdegir): Kubernetes stack python requirements
+# Kubernetes stack can be deployed on instances provisioned from OpenStack, VMs
+# created using libvirt, or on baremetal nodes. The requirements listed below
+# contain all the python packages necessary for Kubernetes stack.
+alembic==1.4.2
+amqp==2.5.2
+asn1crypto==1.3.0
+attrs==19.3.0
+automaton==2.0.0
+Babel==2.8.0
+bcrypt==3.1.7
+beautifulsoup4==4.8.2
+cachetools==4.0.0
+certifi==2019.11.28
+chardet==3.0.4
+click==7.1.1
+cliff==3.0.0
+cmd2==0.8.9
+configparser==5.0.0
+construct==2.8.22
+debtcollector==2.0.0
+decorator==4.4.2
+dictdiffer==0.8.1
+diskimage-builder==2.34.1
+Django==2.2.11
+django-cors-headers==3.2.1
+django-filter==2.2.0
+django-health-check==3.12.1
+djangorestframework==3.11.0
+dnspython==1.15.0
+docker==4.2.0
+dogpile.cache==0.9.0
+dulwich==0.19.15
+dynaconf==2.2.3
+entrypoints==0.3
+eventlet==0.25.1
+extras==1.0.0
+fasteners==0.14.1
+fixtures==3.0.0
+flake8==3.7.9
+Flask==1.1.1
+Flask-Migrate==2.5.3
+Flask-Script==2.0.6
+Flask-SQLAlchemy==2.4.1
+Frozen-Flask==0.15
+futurist==2.1.0
+git-review==1.28.0
+gitdb==4.0.2
+gitdb2==4.0.2
+GitPython==3.1.0
+google-auth==1.12.0
+greenlet==0.4.15
+hacking==2.0.0
+idna==2.9
+ifaddr==0.1.6
+ironic==14.0.0
+ironic-inspector==10.0.0
+ironic-lib==4.1.0
+ironic-python-agent-builder==1.1.0
+iso8601==0.1.12
+itsdangerous==1.1.0
+jmespath==0.9.5
+jsonpatch==1.25
+jsonpath-rw==1.4.0
+jsonpointer==2.0
+jsonschema==3.2.0
+junit-xml==1.9
+keystoneauth1==3.18.0
+keystonemiddleware==8.0.0
+kolla==9.0.1
+kolla-ansible==9.0.1
+kombu==4.6.8
+kubernetes==11.0.0
+libvirt-python==6.1.0
+linecache2==1.0.0
+logutils==0.3.5
+lxml==4.5.0
+Mako==1.1.2
+mccabe==0.6.1
+monotonic==1.5
+more-itertools==8.2.0
+msgpack==0.6.2
+munch==2.5.0
+netaddr==0.7.19
+netifaces==0.10.9
+networkx==2.4
+oauthlib==3.1.0
+openshift==0.11.0
+openstacksdk==0.43.0
+os-client-config==2.0.0
+os-service-types==1.7.0
+os-traits==2.2.0
+osc-lib==2.0.0
+oslo.cache==2.2.0
+oslo.concurrency==4.0.1
+oslo.config==8.0.1
+oslo.context==3.0.0
+oslo.db==8.0.0
+oslo.i18n==4.0.0
+oslo.log==4.1.0
+oslo.messaging==12.0.0
+oslo.middleware==4.0.1
+oslo.policy==3.0.2
+oslo.reports==2.0.0
+oslo.rootwrap==6.0.1
+oslo.serialization==3.1.0
+oslo.service==2.1.0
+oslo.upgradecheck==1.0.0
+oslo.utils==4.1.0
+oslo.versionedobjects==2.0.1
+osprofiler==3.0.0
+paramiko==2.7.1
+Paste==3.4.0
+PasteDeploy==2.1.0
+pecan==1.3.3
+ply==3.11
+prettytable==0.7.2
+proliantutils==2.9.4
+psutil==5.7.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycadf==3.0.0
+pycodestyle==2.5.0
+pycryptodomex==3.9.7
+pyfakefs==4.0.2
+pyflakes==2.1.1
+pyghmi==1.5.13
+Pygments==2.6.1
+pyinotify==0.9.6
+PyMySQL==0.9.3
+PyNaCl==1.3.0
+pyOpenSSL==19.1.0
+pyparsing==2.4.6
+pyperclip==1.7.0
+pyrsistent==0.16.0
+pysendfile==2.0.1
+pysmi==0.3.4
+pysnmp==4.4.12
+python-box==3.4.6
+python-cinderclient==6.0.0
+python-dateutil==2.8.1
+python-dotenv==0.12.0
+python-editor==1.0.4
+python-glanceclient==3.0.0
+python-heatclient==2.0.0
+python-ironic-inspector-client==4.0.0
+python-ironicclient==4.0.0
+python-keystoneclient==3.22.0
+python-mimeparse==1.6.0
+python-neutronclient==7.1.0
+python-novaclient==16.0.0
+python-openstackclient==5.1.0
+python-string-utils==1.0.0
+python-subunit==1.4.0
+python-swiftclient==3.9.0
+pytz==2019.3
+pyzmq==19.0.0
+reno==3.0.0
+repoze.lru==0.7
+requests==2.23.0
+requests-oauthlib==1.3.0
+requestsexceptions==1.4.0
+retrying==1.3.3
+rfc3986==1.3.2
+Routes==2.4.1
+rsa==4.0
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
+shade==1.33.0
+simplegeneric==0.8.1
+simplejson==3.17.0
+smmap==3.0.1
+smmap2==3.0.1
+soupsieve==2.0
+SQLAlchemy==1.3.15
+sqlalchemy-migrate==0.13.0
+sqlparse==0.3.1
+statsd==3.3.0
+stevedore==1.32.0
+sushy==3.1.0
+Tempita==0.5.2
+tenacity==6.1.0
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.4.0
+toml==0.10.0
+tooz==2.2.0
+traceback2==1.4.0
+tzlocal==2.0.0
+unittest2==1.1.0
+urllib3==1.25.8
+vine==1.3.0
+virtualbmc==2.0.0
+virtualenv==20.0.14
+voluptuous==0.11.7
+waitress==1.4.3
+warlock==1.3.3
+wcwidth==0.1.9
+WebOb==1.8.6
+websocket-client==0.57.0
+WebTest==2.0.34
+Werkzeug==1.0.0
+whitenoise==5.0.1
+wrapt==1.12.1
+WSME==0.9.3
+XStatic==1.0.2
+XStatic-Bootstrap-SCSS==3.3.7.1
+XStatic-DataTables==1.10.15.1
+XStatic-jQuery==1.12.4.1
+XStatic-Patternfly==3.21.0.1
+XStatic-Patternfly-Bootstrap-Treeview==2.1.3.2
+yappi==1.2.3
+zeroconf==0.24.5
+
+# vim: set ts=2 sw=2 expandtab:
index 35d5789..732c070 100755 (executable)
@@ -33,4 +33,21 @@ export STACK_ROOT_DIR
 export DEPLOY_SCENARIO="${DEPLOY_SCENARIO:=k8-calico-nofeature}"
 echo "Info  : Deploy scenario is $DEPLOY_SCENARIO"
 
+# NOTE (fdegir): we need to install stack specific operating system requirements
+# and python packages before proceeding further
+echo "Info  : Install stack operating system requirements listed in stack bindep.txt using $PKG_MGR"
+cd "$STACK_ROOT_DIR"
+# bindep -b exits with non-zero if it identifies a missing package so we disable pipefail
+set +o pipefail
+# shellcheck disable=SC2046 disable=SC2086
+bindep -b &> /dev/null || redirect_cmd ${INSTALLER_CMD} $(bindep -b)
+set -o pipefail
+
+# TODO (fdegir): installation of pip packages fail due to not being able to find setuptools
+# as build dependency so we do --no-use-pep517 as workaround.
+# This needs to be revisited later on.
+echo "Info  : Install stack python packages listed in stack requirements.txt using pip"
+echo "        Please note that this may take some time"
+redirect_cmd pip install --no-use-pep517 --force-reinstall -r requirements.txt
+
 # vim: set ts=2 sw=2 expandtab:
index 13213c7..19b0cbd 100644 (file)
@@ -1,10 +1,56 @@
 #-------------------------------------------------------------------------------
-# This file contains engine python requirement version pins.
-# Changing versions might have side effects!
+# This file contains python requirement version pins for tox.
+# Do not modify this file unless you know what you are doing.
+# Modifying this file may result in unexpected behavior.
 #-------------------------------------------------------------------------------
 
-# NOTE: engine test requirements
-tox==3.14.3
+# NOTE (fdegir): tox python requirements
+alabaster==0.7.12
+ansible==2.9.6
 ansible-lint==4.1.0
-yamllint==1.19.0
+appdirs==1.4.4
+Babel==2.8.0
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+cryptography==3.1.1
+distlib==0.3.1
+docutils==0.16
+filelock==3.0.12
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==1.7.0
+importlib-resources==3.0.0
+Jinja2==2.11.2
+MarkupSafe==1.1.1
+packaging==20.4
+pathspec==0.8.0
+pluggy==0.13.1
+py==1.9.0
+pycparser==2.20
+Pygments==2.7.1
+pyparsing==2.4.7
+pytz==2020.1
+PyYAML==5.3.1
+requests==2.24.0
+rtcat-sphinx-theme==0.1.1
+ruamel.yaml==0.16.12
+ruamel.yaml.clib==0.2.2
 shellcheck-py==0.7.0.1
+six==1.15.0
+snowballstemmer==2.0.0
+Sphinx==2.3.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+toml==0.10.1
+tox==3.14.3
+urllib3==1.25.10
+virtualenv==20.0.33
+yamllint==1.19.0
+zipp==3.3.0
+
+# vim: set ts=2 sw=2 expandtab: