Проблема
Я ожидаю, что --symlink-app-data
будет работать при создании virtualenv
Среда
У меня есть для вас образ докера :)
FROM ubuntu:focal
RUN : \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
python3 \
python3-distutils \
curl \
ca-certificates \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV PATH=/venv/bin:$PATH
RUN : \
&& curl --location --silent --output /virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz \
&& python3 /virtualenv.pyz /venv \
&& pip install virtualenv \
&& rm /virtualenv.pyz
ENV \
VIRTUALENV_OVERRIDE_APP_DATA=/opt/virtualenvdata \
VIRTUALENV_SYMLINK_APP_DATA=1
# prepopulate appdata cache
RUN virtualenv /wat --symlink-app-data && rm -rf /wat
USER 1000
RUN : \
&& virtualenv /tmp/test -vvv --with-traceback \
&& ls -al /tmp/test/lib/python3.8/site-packages \
&& /tmp/test/bin/pip install astpretty
Вывод создания виртуальной среды
Обязательно запустите создание с помощью -vvv --with-traceback
:
$ docker build -t test .
STEP 1: FROM ubuntu:focal
STEP 2: RUN : && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-distutils curl ca-certificates && apt-get clean && rm -rf /var/lib/apt/lists/*
--> Using cache cd34708b98bedf0d512db31127ae88edd1288a868a640550a23e3fc125d9c2b1
--> cd34708b98b
STEP 3: ENV PATH=/venv/bin:$PATH
--> Using cache d237cc02ce7a5abed4e67086eb0c245fa4e6406a000a95cf0af1d9209751d162
--> d237cc02ce7
STEP 4: RUN : && curl --location --silent --output /virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz && python3 /virtualenv.pyz /venv && pip install virtualenv && rm /virtualenv.pyz
--> Using cache f0afa26da50e854119e318071290ababf95bae97337a870a04a279e2ce8cdb00
--> f0afa26da50
STEP 5: ENV VIRTUALENV_OVERRIDE_APP_DATA=/opt/virtualenvdata VIRTUALENV_SYMLINK_APP_DATA=1
--> Using cache 4489d1eb9aa51f8a463dffef6cf761258ae5e4ab8667e978fa32af2e70618d6a
--> 4489d1eb9aa
STEP 6: RUN virtualenv /wat --symlink-app-data && rm -rf /wat
--> Using cache 7b1d35bfcc84bc716caa075c655cf566bc0e8f62a9f18f8f0e5e88f2179d2502
--> 7b1d35bfcc8
STEP 7: USER 1000
--> Using cache 6d3a71bdb4a0dad209fbbac17565525e02d99457908a041f8352e72050b91376
--> 6d3a71bdb4a
STEP 8: RUN : && virtualenv /tmp/test -vvv --with-traceback && ls -al /tmp/test/lib/python3.8/site-packages && /tmp/test/bin/pip install astpretty
137 setup logging to NOTSET [DEBUG report:43]
137 app data folder /opt/virtualenvdata has no write access [DEBUG __init__:37]
139 created temporary app data folder /tmp/tmpr8o4y3qh [DEBUG via_tempdir:14]
153 find interpreter for spec PythonSpec(path=/venv/bin/python) [INFO builtin:51]
153 discover exe for PythonInfo(spec=CPython3.8.5.final.0-64, exe=/venv/bin/python, platform=linux, version='3.8.5 (default, Jul 28 2020, 12:59:40) \n[GCC 9.3.0]', encoding_fs_io=utf-8-utf-8) in /usr [DEBUG py_info:371]
154 filesystem is case-sensitive [DEBUG info:28]
159 Attempting to acquire lock 140572762775952 on /tmp/tmpr8o4y3qh/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.lock [DEBUG filelock:270]
159 Lock 140572762775952 acquired on /tmp/tmpr8o4y3qh/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.lock [INFO filelock:274]
160 get interpreter info via cmd: /usr/bin/python3.8 /venv/lib/python3.8/site-packages/virtualenv/discovery/py_info.py [DEBUG cached_py_info:87]
255 wrote python info of /usr/bin/python3.8 at /tmp/tmpr8o4y3qh/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.json [DEBUG via_disk_folder:162]
255 Attempting to release lock 140572762775952 on /tmp/tmpr8o4y3qh/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.lock [DEBUG filelock:315]
255 Lock 140572762775952 released on /tmp/tmpr8o4y3qh/py_info/1/df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6.lock [INFO filelock:318]
255 proposed PythonInfo(spec=CPython3.8.5.final.0-64, system=/usr/bin/python3.8, exe=/venv/bin/python, platform=linux, version='3.8.5 (default, Jul 28 2020, 12:59:40) \n[GCC 9.3.0]', encoding_fs_io=utf-8-utf-8) [INFO builtin:57]
256 accepted PythonInfo(spec=CPython3.8.5.final.0-64, system=/usr/bin/python3.8, exe=/venv/bin/python, platform=linux, version='3.8.5 (default, Jul 28 2020, 12:59:40) \n[GCC 9.3.0]', encoding_fs_io=utf-8-utf-8) [DEBUG builtin:59]
331 create virtual environment via CPython3Posix(dest=/tmp/test, clear=False, global=False) [INFO session:52]
331 create folder /tmp/test/bin [DEBUG _sync:25]
332 create folder /tmp/test/lib/python3.8/site-packages [DEBUG _sync:25]
333 write /tmp/test/pyvenv.cfg [DEBUG pyenv_cfg:34]
333 home = /usr [DEBUG pyenv_cfg:38]
333 implementation = CPython [DEBUG pyenv_cfg:38]
333 version_info = 3.8.5.final.0 [DEBUG pyenv_cfg:38]
333 virtualenv = 20.1.0 [DEBUG pyenv_cfg:38]
333 include-system-site-packages = false [DEBUG pyenv_cfg:38]
333 base-prefix = /usr [DEBUG pyenv_cfg:38]
333 base-exec-prefix = /usr [DEBUG pyenv_cfg:38]
333 base-executable = /usr/bin/python3.8 [DEBUG pyenv_cfg:38]
334 symlink /usr/bin/python3.8 to /tmp/test/bin/python [DEBUG _sync:44]
334 create virtualenv import hook file /tmp/test/lib/python3.8/site-packages/_virtualenv.pth [DEBUG api:95]
335 create /tmp/test/lib/python3.8/site-packages/_virtualenv.py [DEBUG api:98]
336 ============================== target debug ============================== [DEBUG session:54]
336 debug via /tmp/test/bin/python /venv/lib/python3.8/site-packages/virtualenv/create/debug.py [DEBUG creator:213]
336 {
"sys": {
"executable": "/tmp/test/bin/python",
"_base_executable": "/tmp/test/bin/python",
"prefix": "/tmp/test",
"base_prefix": "/usr",
"real_prefix": null,
"exec_prefix": "/tmp/test",
"base_exec_prefix": "/usr",
"path": [
"/usr/lib/python38.zip",
"/usr/lib/python3.8",
"/usr/lib/python3.8/lib-dynload",
"/tmp/test/lib/python3.8/site-packages"
],
"meta_path": [
"<class '_virtualenv._Finder'>",
"<class '_frozen_importlib.BuiltinImporter'>",
"<class '_frozen_importlib.FrozenImporter'>",
"<class '_frozen_importlib_external.PathFinder'>"
],
"fs_encoding": "utf-8",
"io_encoding": "utf-8"
},
"version": "3.8.5 (default, Jul 28 2020, 12:59:40) \n[GCC 9.3.0]",
"makefile_filename": "/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/Makefile",
"os": "<module 'os' from '/usr/lib/python3.8/os.py'>",
"site": "<module 'site' from '/usr/lib/python3.8/site.py'>",
"datetime": "<module 'datetime' from '/usr/lib/python3.8/datetime.py'>",
"math": "<module 'math' (built-in)>",
"json": "<module 'json' from '/usr/lib/python3.8/json/__init__.py'>"
} [DEBUG session:55]
385 add seed packages via FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=symlink, app_data_dir=/tmp/tmpr8o4y3qh) [INFO session:59]
386 install pip from wheel /venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/pip-20.2.4-py2.py3-none-any.whl via SymlinkPipInstall [DEBUG via_app_data:51]
388 install setuptools from wheel /venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/setuptools-50.3.2-py3-none-any.whl via SymlinkPipInstall [DEBUG via_app_data:51]
388 install wheel from wheel /venv/lib/python3.8/site-packages/virtualenv/seed/wheels/embed/wheel-0.35.1-py2.py3-none-any.whl via SymlinkPipInstall [DEBUG via_app_data:51]
389 Attempting to acquire lock 140572759983872 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any.lock [DEBUG filelock:270]
389 Attempting to acquire lock 140572772360448 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any.lock [DEBUG filelock:270]
390 Lock 140572759983872 acquired on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any.lock [INFO filelock:274]
390 build install image for wheel-0.35.1-py2.py3-none-any.whl to /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any [DEBUG base:52]
390 Attempting to acquire lock 140572759984064 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any.lock [DEBUG filelock:270]
390 Lock 140572772360448 acquired on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any.lock [INFO filelock:274]
390 Lock 140572759984064 acquired on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any.lock [INFO filelock:274]
390 build install image for pip-20.2.4-py2.py3-none-any.whl to /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any [DEBUG base:52]
394 build install image for setuptools-50.3.2-py3-none-any.whl to /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any [DEBUG base:52]
513 Attempting to release lock 140572759983872 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any.lock [DEBUG filelock:315]
513 Lock 140572759983872 released on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any.lock [INFO filelock:318]
515 generated console scripts wheel wheel-3.8 wheel3 wheel3.8 [DEBUG base:48]
947 Attempting to release lock 140572759984064 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any.lock [DEBUG filelock:315]
947 Lock 140572759984064 released on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any.lock [INFO filelock:318]
950 generated console scripts easy_install3.8 easy_install easy_install-3.8 easy_install3 [DEBUG base:48]
1712 Attempting to release lock 140572772360448 on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any.lock [DEBUG filelock:315]
1712 Lock 140572772360448 released on /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any.lock [INFO filelock:318]
1715 generated console scripts pip pip3.8 pip-3.8 pip3 [DEBUG base:48]
1715 add activators for Bash, CShell, Fish, PowerShell, Python, Xonsh [INFO session:64]
1719 write /tmp/test/pyvenv.cfg [DEBUG pyenv_cfg:34]
1719 home = /usr [DEBUG pyenv_cfg:38]
1719 implementation = CPython [DEBUG pyenv_cfg:38]
1719 version_info = 3.8.5.final.0 [DEBUG pyenv_cfg:38]
1719 virtualenv = 20.1.0 [DEBUG pyenv_cfg:38]
1719 include-system-site-packages = false [DEBUG pyenv_cfg:38]
1719 base-prefix = /usr [DEBUG pyenv_cfg:38]
1719 base-exec-prefix = /usr [DEBUG pyenv_cfg:38]
1719 base-executable = /usr/bin/python3.8 [DEBUG pyenv_cfg:38]
1719 remove temporary app data folder /tmp/tmpr8o4y3qh [DEBUG via_tempdir:20]
1910 created virtual environment CPython3.8.5.final.0-64 in 1775ms
creator CPython3Posix(dest=/tmp/test, clear=False, global=False)
seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=symlink, app_data_dir=/tmp/tmpr8o4y3qh)
added seed packages: pip==20.2.4, setuptools==50.3.2, wheel==0.35.1
activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator [WARNING __main__:17]
total 76
drwxr-xr-x 3 1000 root 4096 Oct 28 22:58 .
drwxr-xr-x 3 1000 root 4096 Oct 28 22:58 ..
drwxr-xr-x 2 1000 root 4096 Oct 28 22:58 __pycache__
lrwxrwxrwx 1 1000 root 99 Oct 28 22:58 _distutils_hack -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/_distutils_hack
-rw-r--r-- 1 1000 root 18 Oct 28 22:58 _virtualenv.pth
-rw-r--r-- 1 1000 root 5662 Oct 28 22:58 _virtualenv.py
lrwxrwxrwx 1 1000 root 108 Oct 28 22:58 distutils-precedence.pth -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/distutils-precedence.pth
lrwxrwxrwx 1 1000 root 99 Oct 28 22:58 easy_install.py -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/easy_install.py
lrwxrwxrwx 1 1000 root 84 Oct 28 22:58 pip -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any/pip
lrwxrwxrwx 1 1000 root 101 Oct 28 22:58 pip-20.2.4.dist-info -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any/pip-20.2.4.dist-info
lrwxrwxrwx 1 1000 root 102 Oct 28 22:58 pip-20.2.4.virtualenv -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/pip-20.2.4-py2.py3-none-any/pip-20.2.4.virtualenv
lrwxrwxrwx 1 1000 root 97 Oct 28 22:58 pkg_resources -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/pkg_resources
lrwxrwxrwx 1 1000 root 94 Oct 28 22:58 setuptools -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/setuptools
lrwxrwxrwx 1 1000 root 111 Oct 28 22:58 setuptools-50.3.2.dist-info -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/setuptools-50.3.2.dist-info
lrwxrwxrwx 1 1000 root 112 Oct 28 22:58 setuptools-50.3.2.virtualenv -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/setuptools-50.3.2-py3-none-any/setuptools-50.3.2.virtualenv
lrwxrwxrwx 1 1000 root 88 Oct 28 22:58 wheel -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any/wheel
lrwxrwxrwx 1 1000 root 105 Oct 28 22:58 wheel-0.35.1.dist-info -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any/wheel-0.35.1.dist-info
lrwxrwxrwx 1 1000 root 106 Oct 28 22:58 wheel-0.35.1.virtualenv -> /tmp/tmpr8o4y3qh/wheel/3.8/image/1/SymlinkPipInstall/wheel-0.35.1-py2.py3-none-any/wheel-0.35.1.virtualenv
Traceback (most recent call last):
File "/tmp/test/bin/pip", line 5, in <module>
from pip._internal.cli.main import main
ModuleNotFoundError: No module named 'pip'
Error: error building at STEP "RUN : && virtualenv /tmp/test -vvv --with-traceback && ls -al /tmp/test/lib/python3.8/site-packages && /tmp/test/bin/pip install astpretty": error while running runtime: exit status 1
Обратите внимание, что я пытаюсь заполнить кеш, используя пользователя root, а затем, когда непривилегированный пользователь использует этот кеш позже. Я хотел бы использовать это, чтобы ускорить pre-commit.ci на ~ 80-90% для загрузки / скачивания среды python
поэтому я замечаю, что он пропускает appdir, если он не доступен для записи - я хотел бы иметь возможность повторно использовать кеш только для чтения
поэтому я предполагаю, что здесь есть и ошибка, и запрос функции:
Если данные приложения доступны только для чтения, мы должны откатиться к копированию или просто вызвать ошибку?
В идеале я бы хотел, чтобы он выдавал ошибку только в том случае, если ему нужно записать в кеш, но разрешил символическую ссылку в каталог кеша только для чтения
Не стесняйтесь поднять пиар 👍 достижения этого.
круто круто, посмотрю, что я могу сделать - наверное сделаю два прохода (1) исправлю ошибку (2) добавлю фичу