import atexit
import os
import shutil
import stat
import docker
Running duqtools with docker containers
This Tutorial uses docker to create and submit runs which will be analyzed with duqtools.
Getting started
Needed Packages
- The first thing we need is to install duqtools, if you haven't done this yet you can install it via
pip install duqtools - We will also need access to the source of duqtools, it can be downloaded from https://github.com/duqtools/duqtools, This notebook was created with version
1.5.0of duqtools - docker
- We also need access to the following images (or equivalent):
jintrac-imas. which can be found here https://gitlab.eufus.psnc.pl/containerization/jintrac/jintrac-imas-installer
- We also need access to the following images (or equivalent):
- etc...
Some example data
In this tutorial we will use example data, as IMAS is non-free you will have to get your IMAS data through your preferred supplier. We assume that the system is run with JINTRAC (since JINTRAC has a docker container available), and we use a simple JINTRAC template, which you are of course free to substitute for your own
shutil.rmtree('./example', ignore_errors=True)
os.mkdir('example')
# We assume that the duqtools source is available and that we are in the docs/examples folder
shutil.copytree('../../tests/test_data/template_model', 'example/template');
# Some imas data, substitute with your own, as we are not allowed to provide it
shutil.copytree('../../../containerized_runs/imasdb', 'example/imasdb');
# A silly script to support venvs
with open("example/entry.sh", "w") as f:
f.write("""#!/bin/bash
set -e
. /etc/profile.d/modules.sh
module load IMAS
module load fc2k
if [ -d .venv ]; then
. .venv/bin/activate
fi
exec "${@}"
""")
st = os.stat('example/entry.sh')
os.chmod('example/entry.sh', st.st_mode | stat.S_IEXEC)
!chmod +x example/entry.sh
# A simple configuration file for duqtools
with open("example/duqtools.yaml", "w") as f:
f.write("""system:
name: jetto
create:
template: ./template #TODO edit to directory where initial run is stored
template_data:
user: "/example/imasdb"
db: "jet"
shot: 123
run: 1
dimensions:
- operator: multiply
scale_to_error: false
values: [1.1, 1.2, 1.3]
variable: t_e
- operator: multiply
scale_to_error: false
values: [1.1, 1.2, 1.3]
variable: zeff
sampler:
method: latin-hypercube
n_samples: 3
submit:
submit_system: docker
docker_image: jintrac-imas:latest
""")
!tree example #Its okay if this fails if you don't have tree installed
# Get the image and tag it
#!docker login gitlab.eufus.psnc.pl:5050
#!docker pull gitlab.eufus.psnc.pl:5050/containerization/jintrac/jintrac-imas-installer/jintrac-imas:latest
#!docker tag gitlab.eufus.psnc.pl:5050/containerization/jintrac/jintrac-imas-installer/jintrac-imas:latest jintrac-imas:pulled
# Or, build the image yourself
# Follow setup instructions in README.md
!cd ~/local_projects/jintrac-imas-installer && git checkout ab5a6
!cd ~/local_projects/jintrac-imas-installer && git submodule update
!cd ~/local_projects/jintrac-imas-installer && git submodule foreach --recursive git clean -xffd
!cd ~/local_projects/jintrac-imas-installer && DOCKER_BUILDKIT=1 docker build --target jintrac-imas --tag jintrac-imas:ab5a6 . 2> /dev/null
# Do some hotfixing
!echo -e "from jintrac-imas:ab5a6
\
run sed -i 's/mpi4py/mpi4py PyYAML/g' /home/docker/jintrac/build/python/run_python_driver
\
run chmod 777 /home/docker" | docker build -t jintrac-imas:latest -
def niceprint(s):
return print(str(s.output, 'utf-8'))
def run_inside(container, cmd, **kwargs):
return container.exec_run(' '.join(['/example/entry.sh', cmd]), **kwargs)
# clean up existing container
try:
container.stop(timeout=0)
except:
pass
client = docker.from_env()
container = client.containers.run('jintrac-imas:latest',
working_dir="/example",
user=os.getuid(),
detach=True,
entrypoint="/example/entry.sh",
command="tail -f /dev/null",
volumes=[os.getcwd() + "/example:/example"],
auto_remove=True)
atexit.register(lambda: container.stop(timeout=0));
run_inside(container, 'python3 -m venv .venv');
niceprint(run_inside(container, 'python3 -m pip install duqtools -q'))
niceprint(run_inside(container, 'duqtools create --force --yes'))
!cd example && duqtools submit --force --yes
!cd example && duqtools status
execresults = run_inside(container, 'duqtools dash', stream=True)
for x in execresults.output:
print(x.decode())