From 4cfdd1e691faaf201031114c391155b8c3ba3cf1 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Mon, 5 Oct 2020 09:36:12 +0000 Subject: [PATCH] Distribute dependencies This change moves the dependencies required for Kubernetes stack to stack itself, making sure we deal with the dependencies required for it and no other stack. This is achieved by - moving operating system dependencies from engine core bindep.txt into stack bindep.txt - moving python dependencies from engine core requirements.txt into stack requirements.txt - taking setup-stack.sh script in use so engine core executes this script from the stack The benefits of this approach - ability to control dependencies directly within the stack without impacting other stacks and engine core - package dependencies required for the stack itself and not everything - allow identifying engine core dependencies so they can be made part of stack packaging, enabling stacks to stop using leftovers of an earlier deployment done for a different stack - pave the way for multi distro support since some stacks may support more than one distro and that support can be introduced and maintained within corresponding stack - deferred processing of dependencies allow to engine become functioning faster than how it is know. also, the overall deployment time will be shortened a bit with the help of this due to dealing with dependencies based on stack Depends-On: I713d0e333bc4a01c92e6aa6ae1a4e6f3d9efe2ad Change-Id: I9508275fe581cea15af3c9e7dfa0b3342370460b --- bindep.txt | 21 +++ playbooks/roles/package/tasks/pip.yaml | 10 +- requirements.txt | 221 +++++++++++++++++++++++++ setup-stack.sh | 17 ++ test-requirements.txt | 56 ++++++- 5 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 bindep.txt create mode 100644 requirements.txt diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..5aee147 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,21 @@ +#------------------------------------------------------------------------------- +# This file contains operating system package requirements for Kubernetes stack. +# Do not modify this file unless you know what you are doing. +# Modifying this file may result in unexpected behavior. +#------------------------------------------------------------------------------- + +# NOTE (fdegir): Kubernetes stack operating system package requirements +# Kubernetes stack can be deployed on instances created using libvirt thus why +# these packages are installed since we do not know what the target environment +# is in advance, without doing further processing of user input. +libvirt-bin [platform:dpkg] +libvirt-dev [platform:dpkg] +pkg-config [platform:dpkg] +python3-pymysql [platform:dpkg] +python3-zmq [platform:dpkg] +qemu-utils [platform:dpkg] +qemu-kvm [platform:dpkg] +qemu-system-x86 [platform:dpkg] +sgabios [platform:dpkg] + +# vim: set ts=2 sw=2 expandtab: diff --git a/playbooks/roles/package/tasks/pip.yaml b/playbooks/roles/package/tasks/pip.yaml index a4a4642..d403ea7 100644 --- a/playbooks/roles/package/tasks/pip.yaml +++ b/playbooks/roles/package/tasks/pip.yaml @@ -25,12 +25,20 @@ - absent - directory -- name: Download pip packages using requirements.txt file +# NOTE (fdegir): This could perhaps be moved to a different playbook or to engine core +# itself so the collection of core packages are common across stacks +- name: Download engine core pip packages using requirements.txt file command: "pip download -r {{ engine_path }}/requirements.txt --no-cache" changed_when: false args: chdir: "{{ pip_folder }}" +- name: Download stack pip packages using requirements.txt file + command: "pip download -r {{ engine_path }}/engine/stack/{{ stack_type }}/requirements.txt --no-cache" + changed_when: false + args: + chdir: "{{ pip_folder }}" + - name: Copy pip.conf template: src: pip.conf.j2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9c766aa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,221 @@ +#------------------------------------------------------------------------------- +# This file contains python requirement version pins for Kubernetes stack. +# Do not modify this file unless you know what you are doing. +# Modifying this file may result in unexpected behavior. +#------------------------------------------------------------------------------- + +# NOTE (fdegir): Kubernetes stack python requirements +# Kubernetes stack can be deployed on instances provisioned from OpenStack, VMs +# created using libvirt, or on baremetal nodes. The requirements listed below +# contain all the python packages necessary for Kubernetes stack. +alembic==1.4.2 +amqp==2.5.2 +asn1crypto==1.3.0 +attrs==19.3.0 +automaton==2.0.0 +Babel==2.8.0 +bcrypt==3.1.7 +beautifulsoup4==4.8.2 +cachetools==4.0.0 +certifi==2019.11.28 +chardet==3.0.4 +click==7.1.1 +cliff==3.0.0 +cmd2==0.8.9 +configparser==5.0.0 +construct==2.8.22 +debtcollector==2.0.0 +decorator==4.4.2 +dictdiffer==0.8.1 +diskimage-builder==2.34.1 +Django==2.2.11 +django-cors-headers==3.2.1 +django-filter==2.2.0 +django-health-check==3.12.1 +djangorestframework==3.11.0 +dnspython==1.15.0 +docker==4.2.0 +dogpile.cache==0.9.0 +dulwich==0.19.15 +dynaconf==2.2.3 +entrypoints==0.3 +eventlet==0.25.1 +extras==1.0.0 +fasteners==0.14.1 +fixtures==3.0.0 +flake8==3.7.9 +Flask==1.1.1 +Flask-Migrate==2.5.3 +Flask-Script==2.0.6 +Flask-SQLAlchemy==2.4.1 +Frozen-Flask==0.15 +futurist==2.1.0 +git-review==1.28.0 +gitdb==4.0.2 +gitdb2==4.0.2 +GitPython==3.1.0 +google-auth==1.12.0 +greenlet==0.4.15 +hacking==2.0.0 +idna==2.9 +ifaddr==0.1.6 +ironic==14.0.0 +ironic-inspector==10.0.0 +ironic-lib==4.1.0 +ironic-python-agent-builder==1.1.0 +iso8601==0.1.12 +itsdangerous==1.1.0 +jmespath==0.9.5 +jsonpatch==1.25 +jsonpath-rw==1.4.0 +jsonpointer==2.0 +jsonschema==3.2.0 +junit-xml==1.9 +keystoneauth1==3.18.0 +keystonemiddleware==8.0.0 +kolla==9.0.1 +kolla-ansible==9.0.1 +kombu==4.6.8 +kubernetes==11.0.0 +libvirt-python==6.1.0 +linecache2==1.0.0 +logutils==0.3.5 +lxml==4.5.0 +Mako==1.1.2 +mccabe==0.6.1 +monotonic==1.5 +more-itertools==8.2.0 +msgpack==0.6.2 +munch==2.5.0 +netaddr==0.7.19 +netifaces==0.10.9 +networkx==2.4 +oauthlib==3.1.0 +openshift==0.11.0 +openstacksdk==0.43.0 +os-client-config==2.0.0 +os-service-types==1.7.0 +os-traits==2.2.0 +osc-lib==2.0.0 +oslo.cache==2.2.0 +oslo.concurrency==4.0.1 +oslo.config==8.0.1 +oslo.context==3.0.0 +oslo.db==8.0.0 +oslo.i18n==4.0.0 +oslo.log==4.1.0 +oslo.messaging==12.0.0 +oslo.middleware==4.0.1 +oslo.policy==3.0.2 +oslo.reports==2.0.0 +oslo.rootwrap==6.0.1 +oslo.serialization==3.1.0 +oslo.service==2.1.0 +oslo.upgradecheck==1.0.0 +oslo.utils==4.1.0 +oslo.versionedobjects==2.0.1 +osprofiler==3.0.0 +paramiko==2.7.1 +Paste==3.4.0 +PasteDeploy==2.1.0 +pecan==1.3.3 +ply==3.11 +prettytable==0.7.2 +proliantutils==2.9.4 +psutil==5.7.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycadf==3.0.0 +pycodestyle==2.5.0 +pycryptodomex==3.9.7 +pyfakefs==4.0.2 +pyflakes==2.1.1 +pyghmi==1.5.13 +Pygments==2.6.1 +pyinotify==0.9.6 +PyMySQL==0.9.3 +PyNaCl==1.3.0 +pyOpenSSL==19.1.0 +pyparsing==2.4.6 +pyperclip==1.7.0 +pyrsistent==0.16.0 +pysendfile==2.0.1 +pysmi==0.3.4 +pysnmp==4.4.12 +python-box==3.4.6 +python-cinderclient==6.0.0 +python-dateutil==2.8.1 +python-dotenv==0.12.0 +python-editor==1.0.4 +python-glanceclient==3.0.0 +python-heatclient==2.0.0 +python-ironic-inspector-client==4.0.0 +python-ironicclient==4.0.0 +python-keystoneclient==3.22.0 +python-mimeparse==1.6.0 +python-neutronclient==7.1.0 +python-novaclient==16.0.0 +python-openstackclient==5.1.0 +python-string-utils==1.0.0 +python-subunit==1.4.0 +python-swiftclient==3.9.0 +pytz==2019.3 +pyzmq==19.0.0 +reno==3.0.0 +repoze.lru==0.7 +requests==2.23.0 +requests-oauthlib==1.3.0 +requestsexceptions==1.4.0 +retrying==1.3.3 +rfc3986==1.3.2 +Routes==2.4.1 +rsa==4.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +shade==1.33.0 +simplegeneric==0.8.1 +simplejson==3.17.0 +smmap==3.0.1 +smmap2==3.0.1 +soupsieve==2.0 +SQLAlchemy==1.3.15 +sqlalchemy-migrate==0.13.0 +sqlparse==0.3.1 +statsd==3.3.0 +stevedore==1.32.0 +sushy==3.1.0 +Tempita==0.5.2 +tenacity==6.1.0 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.4.0 +toml==0.10.0 +tooz==2.2.0 +traceback2==1.4.0 +tzlocal==2.0.0 +unittest2==1.1.0 +urllib3==1.25.8 +vine==1.3.0 +virtualbmc==2.0.0 +virtualenv==20.0.14 +voluptuous==0.11.7 +waitress==1.4.3 +warlock==1.3.3 +wcwidth==0.1.9 +WebOb==1.8.6 +websocket-client==0.57.0 +WebTest==2.0.34 +Werkzeug==1.0.0 +whitenoise==5.0.1 +wrapt==1.12.1 +WSME==0.9.3 +XStatic==1.0.2 +XStatic-Bootstrap-SCSS==3.3.7.1 +XStatic-DataTables==1.10.15.1 +XStatic-jQuery==1.12.4.1 +XStatic-Patternfly==3.21.0.1 +XStatic-Patternfly-Bootstrap-Treeview==2.1.3.2 +yappi==1.2.3 +zeroconf==0.24.5 + +# vim: set ts=2 sw=2 expandtab: diff --git a/setup-stack.sh b/setup-stack.sh index 35d5789..732c070 100755 --- a/setup-stack.sh +++ b/setup-stack.sh @@ -33,4 +33,21 @@ export STACK_ROOT_DIR export DEPLOY_SCENARIO="${DEPLOY_SCENARIO:=k8-calico-nofeature}" echo "Info : Deploy scenario is $DEPLOY_SCENARIO" +# NOTE (fdegir): we need to install stack specific operating system requirements +# and python packages before proceeding further +echo "Info : Install stack operating system requirements listed in stack bindep.txt using $PKG_MGR" +cd "$STACK_ROOT_DIR" +# bindep -b exits with non-zero if it identifies a missing package so we disable pipefail +set +o pipefail +# shellcheck disable=SC2046 disable=SC2086 +bindep -b &> /dev/null || redirect_cmd ${INSTALLER_CMD} $(bindep -b) +set -o pipefail + +# TODO (fdegir): installation of pip packages fail due to not being able to find setuptools +# as build dependency so we do --no-use-pep517 as workaround. +# This needs to be revisited later on. +echo "Info : Install stack python packages listed in stack requirements.txt using pip" +echo " Please note that this may take some time" +redirect_cmd pip install --no-use-pep517 --force-reinstall -r requirements.txt + # vim: set ts=2 sw=2 expandtab: diff --git a/test-requirements.txt b/test-requirements.txt index 13213c7..19b0cbd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,56 @@ #------------------------------------------------------------------------------- -# This file contains engine python requirement version pins. -# Changing versions might have side effects! +# This file contains python requirement version pins for tox. +# Do not modify this file unless you know what you are doing. +# Modifying this file may result in unexpected behavior. #------------------------------------------------------------------------------- -# NOTE: engine test requirements -tox==3.14.3 +# NOTE (fdegir): tox python requirements +alabaster==0.7.12 +ansible==2.9.6 ansible-lint==4.1.0 -yamllint==1.19.0 +appdirs==1.4.4 +Babel==2.8.0 +certifi==2020.6.20 +cffi==1.14.3 +chardet==3.0.4 +cryptography==3.1.1 +distlib==0.3.1 +docutils==0.16 +filelock==3.0.12 +idna==2.10 +imagesize==1.2.0 +importlib-metadata==1.7.0 +importlib-resources==3.0.0 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +packaging==20.4 +pathspec==0.8.0 +pluggy==0.13.1 +py==1.9.0 +pycparser==2.20 +Pygments==2.7.1 +pyparsing==2.4.7 +pytz==2020.1 +PyYAML==5.3.1 +requests==2.24.0 +rtcat-sphinx-theme==0.1.1 +ruamel.yaml==0.16.12 +ruamel.yaml.clib==0.2.2 shellcheck-py==0.7.0.1 +six==1.15.0 +snowballstemmer==2.0.0 +Sphinx==2.3.1 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +toml==0.10.1 +tox==3.14.3 +urllib3==1.25.10 +virtualenv==20.0.33 +yamllint==1.19.0 +zipp==3.3.0 + +# vim: set ts=2 sw=2 expandtab: -- 2.25.1