From 0292c68abf7407a8b6974f7f9cee5e8714f3e04a Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 13:04:35 +0000 Subject: [PATCH 01/26] add requirements --- Dockerfile | 12 ++----- pip.conf | 5 +++ requirements.txt | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 pip.conf create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index cb104f2..fa0e012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,10 @@ FROM python:3.9.6 +ENV PIP_CONFIG_FILE pip.conf + RUN apt-get update RUN apt-get -y install coinor-cbc -RUN python -m pip install pulp -RUN python -m pip install pydantic -RUN python -m pip install daemonize -RUN python -m pip install sqspy -RUN python -m pip install -U pytest -RUN python -m pip install pytest-cov -RUN python -m pip install pytest-stub -RUN python -m pip install pytest-mock -RUN python -m pip install girth --upgrade +RUN python -m pip install -r requirements.txt RUN mkdir /app WORKDIR /app diff --git a/pip.conf b/pip.conf new file mode 100644 index 0000000..669e4a7 --- /dev/null +++ b/pip.conf @@ -0,0 +1,5 @@ +[global] +index = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +index-url = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +extra-index = https://pypi.python.org/pypi +extra-index-url = https://pypi.python.org/simple/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..754b70e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,91 @@ +annotated-types==0.5.0 +attrs==21.2.0 +Automat==20.2.0 +Babel==2.8.0 +bcrypt==3.2.0 +blinker==1.4 +boto3==1.28.58 +botocore==1.31.58 +certifi==2020.6.20 +chardet==4.0.0 +click==8.0.3 +cloud-init==23.1.2 +colorama==0.4.4 +command-not-found==0.3 +configobj==5.0.6 +constantly==15.1.0 +coverage==7.3.2 +cryptography==3.4.8 +daemonize==2.5.0 +dbus-python==1.2.18 +distro==1.7.0 +distro-info===1.1build1 +exceptiongroup==1.1.3 +httplib2==0.20.2 +hyperlink==21.0.0 +idna==3.3 +importlib-metadata==4.6.4 +incremental==21.3.0 +iniconfig==2.0.0 +jeepney==0.7.1 +Jinja2==3.0.3 +jmespath==1.0.1 +jsonpatch==1.32 +jsonpointer==2.0 +jsonschema==3.2.0 +keyring==23.5.0 +launchpadlib==1.10.16 +lazr.restfulclient==0.14.4 +lazr.uri==1.0.6 +MarkupSafe==2.0.1 +mirth==0.9.0 +more-itertools==8.10.0 +netifaces==0.11.0 +numpy==1.26.0 +oauthlib==3.2.0 +packaging==23.2 +pexpect==4.8.0 +pluggy==1.3.0 +ptyprocess==0.7.0 +PuLP==2.7.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.1 +pydantic==2.4.2 +pydantic_core==2.10.1 +PyGObject==3.42.1 +PyHamcrest==2.0.2 +PyJWT==2.3.0 +pyOpenSSL==21.0.0 +pyparsing==2.4.7 +pyrsistent==0.18.1 +pyserial==3.5 +pytest==7.4.2 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +pytest-stub==1.1.0 +python-apt==2.4.0+ubuntu1 +python-dateutil==2.8.2 +python-debian===0.1.43ubuntu1 +python-magic==0.4.24 +pytz==2022.1 +PyYAML==5.4.1 +requests==2.25.1 +s3transfer==0.7.0 +scipy==1.11.3 +SecretStorage==3.3.1 +service-identity==18.1.0 +six==1.16.0 +sos==4.4 +sqspy==1.0.0 +ssh-import-id==5.11 +systemd-python==234 +tomli==2.0.1 +Twisted==22.1.0 +typing_extensions==4.8.0 +ubuntu-advantage-tools==8001 +ufw==0.36.1 +unattended-upgrades==0.1 +urllib3==1.26.5 +wadllib==1.3.6 +zipp==1.0.0 +zope.interface==5.4.0 From 566ee7bde9ddcf1069d654e6edc72c81c75321ca Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 13:19:23 +0000 Subject: [PATCH 02/26] move command in Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa0e012..9822998 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ ENV PIP_CONFIG_FILE pip.conf RUN apt-get update RUN apt-get -y install coinor-cbc -RUN python -m pip install -r requirements.txt RUN mkdir /app WORKDIR /app @@ -14,4 +13,6 @@ COPY . /app WORKDIR /app/app +RUN python -m pip install -r requirements.txt + CMD [ "python", "main.py" ] From 2204dcc28a35466a1acfc96667855440c4561a00 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 13:24:00 +0000 Subject: [PATCH 03/26] mv cmd --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9822998..aa6f3f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ WORKDIR /app # Bundle app source COPY . /app -WORKDIR /app/app - RUN python -m pip install -r requirements.txt +WORKDIR /app/app + CMD [ "python", "main.py" ] From 0a3dc19f7cd1e268772ba8c88ccb2128f6e2200c Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 13:32:34 +0000 Subject: [PATCH 04/26] add vault secret --- Jenkinsfile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 307591b..75b2f2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,8 +11,17 @@ pipeline { stages { stage('Build Kaniko image') { steps { - withCredentials([usernamePassword(credentialsId: 'DockerHubAccessYardstick', usernameVariable: 'USER', passwordVariable: 'PASS')]) { - container('kaniko') { + withVault([ vaultSecrets: [ + [ + path: "ci/general", + engineVersion: 1, + secretValues: [ + [envVar: 'NEXUS_TOKEN', vaultKey: 'SONATYPE_DEPLOYMENT_USER_TOKEN'] + ] + ] + ]]) { + withCredentials([usernamePassword(credentialsId: 'DockerHubAccessYardstick', usernameVariable: 'USER', passwordVariable: 'PASS')]) { + container('kaniko') { checkout scm // Setup docker credentials sh 'echo "{\\"auths\\":{\\"https://index.docker.io/v1/\\":{\\"auth\\":\\"$(printf "%s:%s" "$USER" "$PASS" | base64 | tr -d \'\n\')\\"}}}" > /kaniko/.docker/config.json' @@ -28,8 +37,9 @@ pipeline { echo ${BRANCH_NAME} | grep -Eo '(release|hotfix)/[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' | grep -Eo '[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' || \ echo ${BRANCH_NAME} | grep -Eo 'YASDEV-([[:digit:]]*)') """ + } + } } - } } } } From d8545a5500e2498140f55fb01f2e5e32429cc07f Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 14:27:18 +0000 Subject: [PATCH 05/26] update build --- .docker-compose/Dockerfile | 16 +++++----------- Dockerfile | 1 + Jenkinsfile | 1 + 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.docker-compose/Dockerfile b/.docker-compose/Dockerfile index 224b14f..d92446f 100644 --- a/.docker-compose/Dockerfile +++ b/.docker-compose/Dockerfile @@ -1,25 +1,19 @@ FROM python:3.9.6 +ENV PIP_CONFIG_FILE pip.conf +ARG NEXUS_TOKEN + RUN apt-get update RUN apt-get -y install coinor-cbc -RUN python -m pip install pulp -RUN python -m pip install pydantic -RUN python -m pip install daemonize -RUN python -m pip install sqspy -RUN python -m pip install -U pytest -RUN python -m pip install pytest-cov -RUN python -m pip install pytest-stub -RUN python -m pip install pytest-mock -RUN python -m pip install girth --upgrade RUN mkdir /app WORKDIR /app -ENV LOCAL_DEV True - # Bundle app source COPY . /app +RUN python -m pip install -r requirements.txt + WORKDIR /app/app CMD [ "python", "main.py" ] diff --git a/Dockerfile b/Dockerfile index aa6f3f8..d92446f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.9.6 ENV PIP_CONFIG_FILE pip.conf +ARG NEXUS_TOKEN RUN apt-get update RUN apt-get -y install coinor-cbc diff --git a/Jenkinsfile b/Jenkinsfile index 75b2f2c..4eff7ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,6 +32,7 @@ pipeline { --insecure=true \ --insecure-registry=docker-registry.default:5000 \ --cache=true \ + --build-arg NEXUS_TOKEN=${NEXUS_TOKEN} \ --cache-repo=docker-registry.default:5000/${REPOSITORY} \ --destination ${env.REPOSITORY}:\$(echo ${BRANCH_NAME} | grep -Eo 'feature/([A-Za-z]+-[0-9]*)' | grep -Eo '[A-Za-z]+-[0-9]*' || \ echo ${BRANCH_NAME} | grep -Eo '(release|hotfix)/[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' | grep -Eo '[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' || \ From 872d8033755fb0f056b90d9da63a951210ac8e3b Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Wed, 4 Oct 2023 18:05:13 +0000 Subject: [PATCH 06/26] moving --- .docker-compose/Dockerfile | 4 +++- Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.docker-compose/Dockerfile b/.docker-compose/Dockerfile index d92446f..c828651 100644 --- a/.docker-compose/Dockerfile +++ b/.docker-compose/Dockerfile @@ -1,8 +1,10 @@ FROM python:3.9.6 -ENV PIP_CONFIG_FILE pip.conf ARG NEXUS_TOKEN +ENV PIP_CONFIG_FILE pip.conf +ENV NEXUS_TOKEN $NEXUS_TOKEN + RUN apt-get update RUN apt-get -y install coinor-cbc diff --git a/Dockerfile b/Dockerfile index d92446f..c828651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ FROM python:3.9.6 -ENV PIP_CONFIG_FILE pip.conf ARG NEXUS_TOKEN +ENV PIP_CONFIG_FILE pip.conf +ENV NEXUS_TOKEN $NEXUS_TOKEN + RUN apt-get update RUN apt-get -y install coinor-cbc From 6d4a643cbd27e00f3fe8580bd8c21af0ccbd7b97 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 15:12:24 +0000 Subject: [PATCH 07/26] mv --- .docker-compose/Dockerfile | 2 +- Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.docker-compose/Dockerfile b/.docker-compose/Dockerfile index c828651..09ece98 100644 --- a/.docker-compose/Dockerfile +++ b/.docker-compose/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.9.6 ARG NEXUS_TOKEN ENV PIP_CONFIG_FILE pip.conf -ENV NEXUS_TOKEN $NEXUS_TOKEN +ENV NEXUS_TOKEN $NEXUS_TOKEN RUN apt-get update RUN apt-get -y install coinor-cbc diff --git a/Dockerfile b/Dockerfile index c828651..a327fa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV PIP_CONFIG_FILE pip.conf -ENV NEXUS_TOKEN $NEXUS_TOKEN +ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ RUN apt-get update RUN apt-get -y install coinor-cbc From 8f818025bcd3d6c697c4ff8858ab10820cce1d15 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 15:19:36 +0000 Subject: [PATCH 08/26] add indexes --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index a327fa5..09ab940 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,9 @@ FROM python:3.9.6 ARG NEXUS_TOKEN +ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ RUN apt-get update From bfcdce8083d6782640def96dddcb2b24b362c458 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 15:30:43 +0000 Subject: [PATCH 09/26] braces --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09ab940..5431210 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV PIP_INDEX https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +ENV PIP_INDEX_URL https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From a3c67c3b45c76e2b6092c50035094c44dfecdeec Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 18:05:59 +0000 Subject: [PATCH 10/26] another way --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5431210..f6c53f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV PIP_INDEX https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV PIP_INDEX https://${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1}:${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +ENV PIP_INDEX_URL https://${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1}:${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From e78d01bec6a5456fe7d5898fdeb2cd4f32672cc5 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 18:12:29 +0000 Subject: [PATCH 11/26] add parens? --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f6c53f7..f396a02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV PIP_INDEX https://${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1}:${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1}:${echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV PIP_INDEX https://${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)}:${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +ENV PIP_INDEX_URL https://${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)}:${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From c14f05fa24933cef4e626683c6d4b3a5cf4bc2ad Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 18:17:03 +0000 Subject: [PATCH 12/26] try --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f396a02..b9f40b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ FROM python:3.9.6 ARG NEXUS_TOKEN - -ENV PIP_INDEX https://${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)}:${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)}:${(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +RUN export PIP_USERNAME=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)` +RUN export PIP_PASSWORD=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)` +ENV PIP_INDEX https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +ENV PIP_INDEX_URL https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From edd8862c68b82ddc346d7e93e1846dda9e0bcc5e Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 18:20:11 +0000 Subject: [PATCH 13/26] try --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b9f40b1..6efe4cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ FROM python:3.9.6 ARG NEXUS_TOKEN +ENV NEXUS_TOKEN $NEXUS_TOKEN RUN export PIP_USERNAME=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)` RUN export PIP_PASSWORD=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)` + ENV PIP_INDEX https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV PIP_INDEX_URL https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From cd9c3eb2b476f5c421b00428cb365efb736f1f33 Mon Sep 17 00:00:00 2001 From: Nick Darrell Date: Thu, 5 Oct 2023 19:34:36 +0000 Subject: [PATCH 14/26] try --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6efe4cd..09ab940 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,9 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV NEXUS_TOKEN $NEXUS_TOKEN -RUN export PIP_USERNAME=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f1)` -RUN export PIP_PASSWORD=`(echo $NEXUS_TOKEN | base64 -d | cut -d ":" -f2)` -ENV PIP_INDEX https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://${PIP_USERNAME}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple +ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi +ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple ENV EXTRA_INDEX https://pypi.python.org/pypi ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ From 040404cc829939c107e514670a8116c6ec4d3ff1 Mon Sep 17 00:00:00 2001 From: brmnjsh Date: Tue, 31 Oct 2023 17:55:54 +0000 Subject: [PATCH 15/26] update mirth version to 0.10.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 754b70e..5256f18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ launchpadlib==1.10.16 lazr.restfulclient==0.14.4 lazr.uri==1.0.6 MarkupSafe==2.0.1 -mirth==0.9.0 +mirth==0.10.2 more-itertools==8.10.0 netifaces==0.11.0 numpy==1.26.0 From ca9438c714f83530611fb5394038099b7e7560c7 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Mon, 20 Nov 2023 15:22:26 -0500 Subject: [PATCH 16/26] edit indexs --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09ab940..3ac7e53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN -ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple -ENV EXTRA_INDEX https://pypi.python.org/pypi -ENV EXTRA_INDEX_URL https://pypi.python.org/simple/ +ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi +ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple RUN apt-get update RUN apt-get -y install coinor-cbc From a6072eb6ebedb5332f58783646a8b92f7a231eb8 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Mon, 20 Nov 2023 15:27:52 -0500 Subject: [PATCH 17/26] remove extra index, fix pip conf --- pip.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pip.conf b/pip.conf index 669e4a7..b347158 100644 --- a/pip.conf +++ b/pip.conf @@ -1,5 +1,3 @@ [global] -index = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/pypi -index-url = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi-internal/simple -extra-index = https://pypi.python.org/pypi -extra-index-url = https://pypi.python.org/simple/ \ No newline at end of file +index = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi +index-url = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple \ No newline at end of file From f2324c26136362f408932dd482c1bec071a16c71 Mon Sep 17 00:00:00 2001 From: brmnjsh Date: Wed, 29 Nov 2023 23:18:59 +0000 Subject: [PATCH 18/26] get requirements.txt working --- .docker-compose/Dockerfile | 3 +- .vagrantrc | 48 --------------------- Dockerfile | 23 ++++++---- app/lib/irt/models/rasch.py | 2 +- pip.conf | 5 ++- rc | 9 ---- requirements.txt | 84 +------------------------------------ 7 files changed, 22 insertions(+), 152 deletions(-) delete mode 100644 .vagrantrc delete mode 100644 rc diff --git a/.docker-compose/Dockerfile b/.docker-compose/Dockerfile index 09ece98..3fb53cd 100644 --- a/.docker-compose/Dockerfile +++ b/.docker-compose/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.9.6 ARG NEXUS_TOKEN ENV PIP_CONFIG_FILE pip.conf -ENV NEXUS_TOKEN $NEXUS_TOKEN +ENV PYTHONPATH /pip +ENV NEXUS_TOKEN $NEXUS_TOKEN RUN apt-get update RUN apt-get -y install coinor-cbc diff --git a/.vagrantrc b/.vagrantrc deleted file mode 100644 index ddc7b4d..0000000 --- a/.vagrantrc +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -. $MEASURE_SRC_PATH/.vagrantrc - -[ ! -f docker-compose.override.yml ] && cat < docker-compose.override.yml -override_file_placeholder: - image: busybox -YAML - -measure-vagrant-id() { - vagrant global-status | grep 'yardstick\s*$' | awk '{ print $1 }' -} - -vagrant-shell() { - vagrant ssh `measure-vagrant-id` -} - -vagrant-ssh() { - vagrant ssh `measure-vagrant-id` -c "$*" -} - -compose() { - vagrant-ssh "cd /apps/measure-solver && \\ - docker-compose -f '/app/docker-compose.yml' \\ - -f '/apps/measure-solver/docker-compose.yml' \\ - -f '/apps/measure-solver/docker-compose.override.yml' \\ - $*" -} - -run() { - compose "run --rm measure-solver $*" -} - -run-script() { - run "rails $*" -} - -measure-solver-logs() { - compose "logs measure-solver" -} - -restart-measure-solver() { - compose "restart measure-solver" -} - -rails() { - run-script "$*" -} diff --git a/Dockerfile b/Dockerfile index 3ac7e53..64b0312 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,15 @@ -FROM python:3.9.6 +FROM python:3.9.6 AS base +LABEL maintainer='Meazure Developers ' ARG NEXUS_TOKEN +ARG NEXUS_USER +ARG NEXUS_PASS -ENV PIP_INDEX https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi -ENV PIP_INDEX_URL https://$NEXUS_TOKEN@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple +ENV PIP_CONFIG_FILE pip.conf +ENV PYTHONPATH /pip +ENV NEXUS_TOKEN $NEXUS_TOKEN +ENV NEXUS_USER $NEXUS_USER +ENV NEXUS_PASS $NEXUS_PASS RUN apt-get update RUN apt-get -y install coinor-cbc @@ -11,11 +17,12 @@ RUN apt-get -y install coinor-cbc RUN mkdir /app WORKDIR /app -# Bundle app source +CMD [ "python", "app/main.py" ] + +FROM base AS development + +FROM base + COPY . /app RUN python -m pip install -r requirements.txt - -WORKDIR /app/app - -CMD [ "python", "main.py" ] diff --git a/app/lib/irt/models/rasch.py b/app/lib/irt/models/rasch.py index 96090b5..624e472 100644 --- a/app/lib/irt/models/rasch.py +++ b/app/lib/irt/models/rasch.py @@ -1,4 +1,4 @@ -from girth import ability_mle +from mirth import ability_mle from lib.irt.models.base import * diff --git a/pip.conf b/pip.conf index b347158..5e54d91 100644 --- a/pip.conf +++ b/pip.conf @@ -1,3 +1,4 @@ [global] -index = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi -index-url = https://${NEXUS_TOKEN}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple \ No newline at end of file +index = https://gDFaNrUq:GD7Y8tLmE6wYMZAE_JC-SWcImC7TLHB2ut_WLL4w8hBC@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi +index-url = https://gDFaNrUq:GD7Y8tLmE6wYMZAE_JC-SWcImC7TLHB2ut_WLL4w8hBC@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple +target = /pip diff --git a/rc b/rc deleted file mode 100644 index f4b4dd2..0000000 --- a/rc +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -export MEASURE_SRC_PATH=../yardstick - -if [ -f .user_rc ]; then - . .user_rc -fi - -source .vagrantrc diff --git a/requirements.txt b/requirements.txt index 5256f18..8803345 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,91 +1,9 @@ -annotated-types==0.5.0 -attrs==21.2.0 -Automat==20.2.0 -Babel==2.8.0 -bcrypt==3.2.0 -blinker==1.4 -boto3==1.28.58 -botocore==1.31.58 -certifi==2020.6.20 -chardet==4.0.0 -click==8.0.3 -cloud-init==23.1.2 -colorama==0.4.4 -command-not-found==0.3 -configobj==5.0.6 -constantly==15.1.0 -coverage==7.3.2 -cryptography==3.4.8 daemonize==2.5.0 -dbus-python==1.2.18 -distro==1.7.0 -distro-info===1.1build1 -exceptiongroup==1.1.3 -httplib2==0.20.2 -hyperlink==21.0.0 -idna==3.3 -importlib-metadata==4.6.4 -incremental==21.3.0 -iniconfig==2.0.0 -jeepney==0.7.1 -Jinja2==3.0.3 -jmespath==1.0.1 -jsonpatch==1.32 -jsonpointer==2.0 -jsonschema==3.2.0 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -MarkupSafe==2.0.1 -mirth==0.10.2 -more-itertools==8.10.0 -netifaces==0.11.0 -numpy==1.26.0 -oauthlib==3.2.0 -packaging==23.2 -pexpect==4.8.0 -pluggy==1.3.0 -ptyprocess==0.7.0 +mirth==0.10.3 PuLP==2.7.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.1 pydantic==2.4.2 -pydantic_core==2.10.1 -PyGObject==3.42.1 -PyHamcrest==2.0.2 -PyJWT==2.3.0 -pyOpenSSL==21.0.0 -pyparsing==2.4.7 -pyrsistent==0.18.1 -pyserial==3.5 pytest==7.4.2 pytest-cov==4.1.0 pytest-mock==3.11.1 pytest-stub==1.1.0 -python-apt==2.4.0+ubuntu1 -python-dateutil==2.8.2 -python-debian===0.1.43ubuntu1 -python-magic==0.4.24 -pytz==2022.1 -PyYAML==5.4.1 -requests==2.25.1 -s3transfer==0.7.0 -scipy==1.11.3 -SecretStorage==3.3.1 -service-identity==18.1.0 -six==1.16.0 -sos==4.4 sqspy==1.0.0 -ssh-import-id==5.11 -systemd-python==234 -tomli==2.0.1 -Twisted==22.1.0 -typing_extensions==4.8.0 -ubuntu-advantage-tools==8001 -ufw==0.36.1 -unattended-upgrades==0.1 -urllib3==1.26.5 -wadllib==1.3.6 -zipp==1.0.0 -zope.interface==5.4.0 From 0675a634fba0e9f7f408fe3476714f6277fbe7e5 Mon Sep 17 00:00:00 2001 From: brmnjsh Date: Wed, 29 Nov 2023 23:19:51 +0000 Subject: [PATCH 19/26] remove .docker-compose dockerfile --- .docker-compose/Dockerfile | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .docker-compose/Dockerfile diff --git a/.docker-compose/Dockerfile b/.docker-compose/Dockerfile deleted file mode 100644 index 3fb53cd..0000000 --- a/.docker-compose/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.9.6 - -ARG NEXUS_TOKEN - -ENV PIP_CONFIG_FILE pip.conf -ENV PYTHONPATH /pip -ENV NEXUS_TOKEN $NEXUS_TOKEN - -RUN apt-get update -RUN apt-get -y install coinor-cbc - -RUN mkdir /app -WORKDIR /app - -# Bundle app source -COPY . /app - -RUN python -m pip install -r requirements.txt - -WORKDIR /app/app - -CMD [ "python", "main.py" ] From 04107d5be525fd31012e4f57945f62abdd836844 Mon Sep 17 00:00:00 2001 From: brmnjsh Date: Thu, 30 Nov 2023 02:41:59 +0000 Subject: [PATCH 20/26] try without requirements.txt --- Dockerfile | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64b0312..bdac2df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,18 @@ FROM python:3.9.6 AS base LABEL maintainer='Meazure Developers ' -ARG NEXUS_TOKEN -ARG NEXUS_USER -ARG NEXUS_PASS - -ENV PIP_CONFIG_FILE pip.conf -ENV PYTHONPATH /pip -ENV NEXUS_TOKEN $NEXUS_TOKEN -ENV NEXUS_USER $NEXUS_USER -ENV NEXUS_PASS $NEXUS_PASS - RUN apt-get update RUN apt-get -y install coinor-cbc +ARG NEXUS_TOKEN + +# ENV PIP_CONFIG_FILE pip.conf +ENV PYTHONPATH /pip +ENV NEXUS_TOKEN $NEXUS_TOKEN +ENV PIP_DEFAULT_INDEX_URL https://artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple/:_auth=${NEXUS_TOKEN} +ENV PIP_DEFAULT_INDEX https://artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi/:_auth=${NEXUS_TOKEN} +ENV PIP_DEFAULT_TARGET /pip + RUN mkdir /app WORKDIR /app From 1d38b19adb1650cbeb08a2c2ebd53223ce9396f2 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 30 Nov 2023 12:09:30 -0500 Subject: [PATCH 21/26] set index and index url in secrest env --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index bdac2df..c9b229b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,13 @@ LABEL maintainer='Meazure Developers ' RUN apt-get update RUN apt-get -y install coinor-cbc -ARG NEXUS_TOKEN +ARG PIP_INDEX_URL +ARG PIP_INDEX -# ENV PIP_CONFIG_FILE pip.conf -ENV PYTHONPATH /pip -ENV NEXUS_TOKEN $NEXUS_TOKEN -ENV PIP_DEFAULT_INDEX_URL https://artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple/:_auth=${NEXUS_TOKEN} -ENV PIP_DEFAULT_INDEX https://artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi/:_auth=${NEXUS_TOKEN} +ENV PIP_INDEX_URL $PIP_INDEX_URL +ENV PIP_INDEX $PIP_INDEX ENV PIP_DEFAULT_TARGET /pip +ENV PYTHONPATH /pip RUN mkdir /app WORKDIR /app From 27abee53a3e1ed79145cf35f8b5ddbb386e3b9f6 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 30 Nov 2023 12:17:38 -0500 Subject: [PATCH 22/26] target not default target for env var --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c9b229b..9ae35da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ARG PIP_INDEX ENV PIP_INDEX_URL $PIP_INDEX_URL ENV PIP_INDEX $PIP_INDEX -ENV PIP_DEFAULT_TARGET /pip +ENV PIP_TARGET /pip ENV PYTHONPATH /pip RUN mkdir /app From 8f89ad11691aed12b8d607bf84a97918aca0af57 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 30 Nov 2023 13:04:31 -0500 Subject: [PATCH 23/26] pip.conf no longer needed --- pip.conf | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 pip.conf diff --git a/pip.conf b/pip.conf deleted file mode 100644 index 5e54d91..0000000 --- a/pip.conf +++ /dev/null @@ -1,4 +0,0 @@ -[global] -index = https://gDFaNrUq:GD7Y8tLmE6wYMZAE_JC-SWcImC7TLHB2ut_WLL4w8hBC@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi -index-url = https://gDFaNrUq:GD7Y8tLmE6wYMZAE_JC-SWcImC7TLHB2ut_WLL4w8hBC@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple -target = /pip From f1718eae3754633b6d795af6e4298f4215a12883 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 1 Dec 2023 11:26:13 -0500 Subject: [PATCH 24/26] new build args and secret values --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4eff7ca..1195cf6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,8 @@ pipeline { path: "ci/general", engineVersion: 1, secretValues: [ - [envVar: 'NEXUS_TOKEN', vaultKey: 'SONATYPE_DEPLOYMENT_USER_TOKEN'] + [envVar: 'PIP_INDEX_URL', vaultKey: 'SONATYPE_DEPLOYMENT_PIP_INDEX_URL'], + [envVar: 'PIP_INDEX', vaultKey: 'SONATYPE_DEPLOYMENT_PIP_INDEX'] ] ] ]]) { @@ -32,7 +33,8 @@ pipeline { --insecure=true \ --insecure-registry=docker-registry.default:5000 \ --cache=true \ - --build-arg NEXUS_TOKEN=${NEXUS_TOKEN} \ + --build-arg PIP_INDEX_URL=${PIP_INDEX_URL} \ + --build-arg PIP_INDEX=${PIP_INDEX} \ --cache-repo=docker-registry.default:5000/${REPOSITORY} \ --destination ${env.REPOSITORY}:\$(echo ${BRANCH_NAME} | grep -Eo 'feature/([A-Za-z]+-[0-9]*)' | grep -Eo '[A-Za-z]+-[0-9]*' || \ echo ${BRANCH_NAME} | grep -Eo '(release|hotfix)/[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' | grep -Eo '[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' || \ From 55f71d6b81bf48d8ad2cdf720460462345262ee1 Mon Sep 17 00:00:00 2001 From: Taylor Christie <74208532+tchristie-meazure@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:22:08 -0700 Subject: [PATCH 25/26] Update Jenkinsfile --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1195cf6..23f96a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,8 +16,8 @@ pipeline { path: "ci/general", engineVersion: 1, secretValues: [ - [envVar: 'PIP_INDEX_URL', vaultKey: 'SONATYPE_DEPLOYMENT_PIP_INDEX_URL'], - [envVar: 'PIP_INDEX', vaultKey: 'SONATYPE_DEPLOYMENT_PIP_INDEX'] + [envVar: 'PIP_USER', vaultKey: 'SONATYPE_DEPLOYMENT_USER'], + [envVar: 'PIP_PASSWORD', vaultKey: 'SONATYPE_DEPLOYMENT_USER_PASSCODE'] ] ] ]]) { @@ -33,8 +33,8 @@ pipeline { --insecure=true \ --insecure-registry=docker-registry.default:5000 \ --cache=true \ - --build-arg PIP_INDEX_URL=${PIP_INDEX_URL} \ - --build-arg PIP_INDEX=${PIP_INDEX} \ + --build-arg PIP_INDEX_URL=https://${PIP_USER}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/simple \ + --build-arg PIP_INDEX=https://${PIP_USER}:${PIP_PASSWORD}@artifact.it.getyardstick.com/repository/meazurelearning-pypi/pypi \ --cache-repo=docker-registry.default:5000/${REPOSITORY} \ --destination ${env.REPOSITORY}:\$(echo ${BRANCH_NAME} | grep -Eo 'feature/([A-Za-z]+-[0-9]*)' | grep -Eo '[A-Za-z]+-[0-9]*' || \ echo ${BRANCH_NAME} | grep -Eo '(release|hotfix)/[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' | grep -Eo '[[:digit:]]+\\.[[:digit:]]+\\.[[:digit:]]+' || \ From b5f1b883993854b8c325f68cf8cc9d656dc28cfc Mon Sep 17 00:00:00 2001 From: Taylor Christie <74208532+tchristie-meazure@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:25:10 -0700 Subject: [PATCH 26/26] Update Jenkinsfile --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 23f96a6..7bf2664 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,8 +16,8 @@ pipeline { path: "ci/general", engineVersion: 1, secretValues: [ - [envVar: 'PIP_USER', vaultKey: 'SONATYPE_DEPLOYMENT_USER'], - [envVar: 'PIP_PASSWORD', vaultKey: 'SONATYPE_DEPLOYMENT_USER_PASSCODE'] + [envVar: 'PIP_USER', vaultKey: 'SONATYPE_READONLY_USER'], + [envVar: 'PIP_PASSWORD', vaultKey: 'SONATYPE_READONLY_USER_PASSCODE'] ] ] ]]) {