如何打包训练环境 (conda pack)

• 打包环境

root@autodl-container-f4d5458449-4ea6d9e7:~# conda create -n llm_env --clone base
Source:      /root/miniconda3
Destination: /root/miniconda3/envs/llm_env
The following packages cannot be cloned out of the root environment:
 - defaults/linux-64::conda-4.10.3-py38h06a4308_0
Packages: 34
Files: 35817
Preparing transaction: done
Verifying transaction: done
Executing transaction: done
#
# To activate this environment, use
#
#     $ conda activate llm_env
#
# To deactivate an active environment, use
#
#     $ conda deactivate

root@autodl-container-f4d5458449-4ea6d9e7:~# conda env list
# conda environments:
#
base                  *  /root/miniconda3
llm_env                    /root/miniconda3/envs/llm_env

• conda pack

# 安装conda pack
conda install conda-pack

RemoveError: ‘requests’ is a dependency of conda and cannot be removed from conda’s operating environment.
conda update --force conda

CondaHTTPError: HTTP 429 TOO MANY REQUESTS for url <https://mirrors.ustc.edu.cn/anaconda/pkgs/main/linux-64/current_repodata.json> Elapsed: 00:46.163959

step 1:
root@autodl-container-f4d5458449-4ea6d9e7:~# vim ~/.condarc
root@autodl-container-f4d5458449-4ea6d9e7:~# cat ~/.condarc
auto_activate_base: false
show_channel_urls: true
ssl_verify: false
channels:
  - conda-forge
  - <http://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/>
  - <http://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/>
  - <http://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/msys2/>
  - <https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/msys2/>
  - <https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge>
  - <https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/>
  - defaults

step 2:
conda create --name superset python=3.8

# 打包环境
conda pack -n llm_env

root@autodl-container-30da119afa-edacec3e:~# conda pack -n llm_env
Collecting packages...
CondaPackError: Cannot pack an environment with editable packages
installed (e.g. from `python setup.py develop` or
 `pip install -e`). Editable packages found:

- /root/autodl-tmp/peft/src

conda pack -n llm_env --ignore-editable-packages

• 安装环境

# 在跳板机进行上传
scp llm_env.tar.gz lanlong@172.20.208.8:~/zyliang

# 与向上传相应的, 补充一个向下取的例子; 但是跳板机比较特殊, 将文件传到跳板机需要用FileZilla
scp -rP 35381 root@region-9.autodl.pro:~/llm_env.tar.gz ~/zyliang

# 在目标服务器进行安装
mkdir -p llm_env

(base) lanlong@a101:~/zyliang$ tar -xzf llm_env.tar.gz -C llm_env
(base) lanlong@a101:~/zyliang$ cd llm_env
(base) lanlong@a101:~/zyliang/llm_env$ ls
bin              conda-meta  include  lib    ssl                          x86_64-conda-linux-gnu
compiler_compat  etc         info     share  x86_64-conda_cos6-linux-gnu

./llm_env/bin/python # 测试
ctrl + z 退出
source llm_env/bin/activate # 激活环境, llm_env/bin 会被添加到环境变量
python # 在环境中运行python
conda-unpack # 在激活环境中清楚前缀
ipython --version

# 若不再使用
source llm_env/bin/deactivate # 停用环境并将其从环境变量中删除

• Ubuntu更换镜像

cd /etc/apt/
sudo cp sources.list sources.list.old
sudo vim /etc/apt/sources.list

deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ trusty main universe restricted multiverse
deb http://archive.ubuntu.com/ubuntu/ trusty main universe restricted multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse