From 9f0e9a1d1fdb320335c656aef3411132d12e1ba0 Mon Sep 17 00:00:00 2001 From: baominghelly <1508269885@qq.com> Date: Wed, 17 Sep 2025 16:33:19 +0800 Subject: [PATCH] Change dataset directory name and add base dataset directory env variable --- Detection/data/VOCdevkit | 1 - Detection/data/coco | 2 +- Detection/fasterrcnn/run_train.sh | 2 +- Detection/fastrccn/fastrcnn.py | 12 ++++++---- Detection/ssd/detect.py | 6 ++++- Detection/ssd/utils.py | 5 ++++ Detection/yolo/READEME.md | 2 +- GAN/data/lsun | 1 - ImageClassification/data/imagenet2012 | 1 - NLP/HuggingFace/run_eval_offline.sh | 2 +- NLP/HuggingFace/run_train_offline.sh | 2 +- NLP/data/squad | 1 - README.md | 26 +++++++++++++++------ Recommendation/DLRM/run_train.sh | 2 +- SR/ESPCN/data_utils.py | 8 +++++-- SR/data/VOC2012-ESPCN | 1 - Segmentation/data/VOCdevkit | 2 +- Segmentation/deeplab/run_eval.sh | 2 +- Segmentation/deeplab/run_train.sh | 2 +- Segmentation/fcn/run_eval.sh | 2 +- Segmentation/fcn/run_train.sh | 2 +- Segmentation/lraspp/run_eval.sh | 2 +- Segmentation/lraspp/run_train.sh | 2 +- Segmentation/unet/run_eval.sh | 2 +- Segmentation/unet/run_train.sh | 2 +- Speech/data/LibriSpeech | 2 +- Speech/data/data_thchs30 | 2 +- Speech/deepspeech2/run_eval.sh | 2 +- Speech/deepspeech2/run_train.sh | 2 +- Speech/wav2vec/run_eval_offline.sh | 2 +- Speech/wav2vec/run_eval_online.sh | 2 +- Speech/wav2vec/run_train_offline.sh | 2 +- Speech/wav2vec/run_train_online.sh | 2 +- TimeSeriesPrediction/data/complete_data.csv | 1 - TimeSeriesPrediction/data/mnist | 1 - TimeSeriesPrediction/lstm/run_eval.sh | 2 +- TimeSeriesPrediction/lstm/run_train.sh | 2 +- TimeSeriesPrediction/tcn/pmnist_test.py | 6 ++++- env.sh | 2 ++ run_train_all.sh | 14 +++++------ 40 files changed, 82 insertions(+), 54 deletions(-) delete mode 120000 Detection/data/VOCdevkit delete mode 120000 GAN/data/lsun delete mode 120000 ImageClassification/data/imagenet2012 delete mode 120000 NLP/data/squad delete mode 120000 SR/data/VOC2012-ESPCN delete mode 120000 TimeSeriesPrediction/data/complete_data.csv delete mode 120000 TimeSeriesPrediction/data/mnist diff --git a/Detection/data/VOCdevkit b/Detection/data/VOCdevkit deleted file mode 120000 index df9db1f..0000000 --- a/Detection/data/VOCdevkit +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/VOCdevkit \ No newline at end of file diff --git a/Detection/data/coco b/Detection/data/coco index 1865170..0f8589a 120000 --- a/Detection/data/coco +++ b/Detection/data/coco @@ -1 +1 @@ -/data1/shared/Dataset/coco/ \ No newline at end of file +/data-aisoft/Dataset/coco \ No newline at end of file diff --git a/Detection/fasterrcnn/run_train.sh b/Detection/fasterrcnn/run_train.sh index a2f03ba..333f04b 100644 --- a/Detection/fasterrcnn/run_train.sh +++ b/Detection/fasterrcnn/run_train.sh @@ -5,7 +5,7 @@ set -e export CUDA_VISIBLE_DEVICES=0 # 使用环境变量，如果没有提供则使用默认路径 -data_dir=${DATA_DIR:-""} # data1/shared/Dataset/VOCdevkit +data_dir=${DATA_DIR:-""} # ${BASE_DATASET_DIR}/VOCdevkit # 确保数据集路径存在 if [ ! -d "$DATA_DIR" ]; then diff --git a/Detection/fastrccn/fastrcnn.py b/Detection/fastrccn/fastrcnn.py index b4d7cd2..e820fcc 100644 --- a/Detection/fastrccn/fastrcnn.py +++ b/Detection/fastrccn/fastrcnn.py @@ -102,9 +102,13 @@ def get_coco_loader(root, ann_file, transform, batch_size, sample_percentage, sh ) return loader +base_dataset_dir = os.getenv('BASE_DATASET_DIR') +if base_dataset_dir is None: + raise ValueError("BASE_DATASET_DIR environment variable not set!") + train_loader = get_coco_loader( - root="/data1/shared/Dataset/coco/images/train2017", - ann_file="/data1/shared/Dataset/coco/images/annotations/instances_train2017.json", + root=os.path.join(base_dataset_dir, "coco/images/train2017"), + ann_file=os.path.join(base_dataset_dir, "coco/images/annotations/instances_train2017.json"), transform=transform, batch_size=args.train_batch, sample_percentage=0.5, # 设置抽样比例为 50% @@ -112,8 +116,8 @@ def get_coco_loader(root, ann_file, transform, batch_size, sample_percentage, sh ) test_loader = get_coco_loader( - root="/data1/shared/Dataset/coco/images/val2017", - ann_file="/data1/shared/Dataset/coco/images/annotations/instances_val2017.json", + root=os.path.join(base_dataset_dir, "coco/images/val2017"), + ann_file=os.path.join(base_dataset_dir, "coco/images/annotations/instances_val2017.json"), transform=transform, batch_size=args.infer_batch, sample_percentage=args.sample, diff --git a/Detection/ssd/detect.py b/Detection/ssd/detect.py index ef81463..e578364 100644 --- a/Detection/ssd/detect.py +++ b/Detection/ssd/detect.py @@ -1,6 +1,7 @@ from torchvision import transforms from utils import * from PIL import Image, ImageDraw, ImageFont +import os device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -96,7 +97,10 @@ def detect(original_image, min_score, max_overlap, top_k, suppress=None): if __name__ == '__main__': - img_path = '/data1/shared/Dataset/VOC2007/JPEGImages/000001.jpg' + base_dataset_dir = os.getenv('BASE_DATASET_DIR') + if base_dataset_dir is None: + raise ValueError("BASE_DATASET_DIR environment variable not set!") + img_path = os.path.join(base_dataset_dir, 'VOC2007/JPEGImages/000001.jpg') original_image = Image.open(img_path, mode='r') original_image = original_image.convert('RGB') detect(original_image, min_score=0.2, max_overlap=0.5, top_k=200).show() diff --git a/Detection/ssd/utils.py b/Detection/ssd/utils.py index f2aa6a4..313f246 100644 --- a/Detection/ssd/utils.py +++ b/Detection/ssd/utils.py @@ -57,6 +57,11 @@ def create_data_lists(voc07_path, voc12_path, output_folder): :param voc12_path: path to the 'VOC2012' folder :param output_folder: folder where the JSONs must be saved """ + + if not os.path.exists(output_folder): + os.makedirs(output_folder) + print(f"Created output folder: {output_folder}") + voc07_path = os.path.abspath(voc07_path) voc12_path = os.path.abspath(voc12_path) diff --git a/Detection/yolo/READEME.md b/Detection/yolo/READEME.md index 012a00e..ccbc9bc 100644 --- a/Detection/yolo/READEME.md +++ b/Detection/yolo/READEME.md @@ -7,7 +7,7 @@ pip install -r requirements.txt ``` mkdir datasets cd datasets -ln -s /data1/shared/Dataset/coco/ ./ +ln -s ${BASE_DATASET_DIR}/coco/ ./ ``` # 训练 diff --git a/GAN/data/lsun b/GAN/data/lsun deleted file mode 120000 index 3dfe23a..0000000 --- a/GAN/data/lsun +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/lsun/ \ No newline at end of file diff --git a/ImageClassification/data/imagenet2012 b/ImageClassification/data/imagenet2012 deleted file mode 120000 index aa2f81d..0000000 --- a/ImageClassification/data/imagenet2012 +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/imagenet2012 \ No newline at end of file diff --git a/NLP/HuggingFace/run_eval_offline.sh b/NLP/HuggingFace/run_eval_offline.sh index 8ee907f..ee5b83b 100644 --- a/NLP/HuggingFace/run_eval_offline.sh +++ b/NLP/HuggingFace/run_eval_offline.sh @@ -7,7 +7,7 @@ if [ -e "../data/squad" ]; then echo "../data/squad exists" else # 创建软连接 - ln -s /data1/shared/Dataset/squad ../data/squad + ln -s ${BASE_DATASET_DIR}/squad ../data/squad fi export SQUAD_PATH="../data/squad" diff --git a/NLP/HuggingFace/run_train_offline.sh b/NLP/HuggingFace/run_train_offline.sh index abac807..50332c1 100755 --- a/NLP/HuggingFace/run_train_offline.sh +++ b/NLP/HuggingFace/run_train_offline.sh @@ -7,7 +7,7 @@ if [ -e "../data/squad" ]; then echo "../data/squad exists" else # 创建软连接 - ln -s /data1/shared/Dataset/squad ../data/squad + ln -s ${BASE_DATASET_DIR}/squad ../data/squad fi export SQUAD_PATH="../data/squad" diff --git a/NLP/data/squad b/NLP/data/squad deleted file mode 120000 index 4b3be36..0000000 --- a/NLP/data/squad +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/squad \ No newline at end of file diff --git a/README.md b/README.md index 9e4f95c..389c9a0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,32 @@ # PyTorchModels -## 项目结构 +## Project Structure + ``` PyTorchModels \ - - ImageClassification \ - - NLP \ - - Detection \ - - Segmentation + - ImageClassification \ + - NLP \ + - Detection \ + - Segmentation ``` +## Prerequisite + +1. Change platform name and dataset directory in `env.sh` file. + +## How to use this repo in NVIDIA GPU platform? + +1. `source env.sh` +2. `nohup bash run_train_all.sh >> output.log 2>&1 &` + ## How to use this repo in Ascend/Cambricon hardware platform? -Test is performed in docker for this two platform. +Test is performed in docker for this two platform. ### Important: Before test, you need to link the right dataset paths to our repo! ### Ascend platform + 1. init Ascend toolkit by `source /usr/local/Ascend/ascend-toolkit/set_env.sh` 2. Replace the right paltform name in env script and init platform env variable by `source env.sh` 3. Add our `usercustomize.py` path to PYTHONPATH by `export PYTHONPATH=$PYTHONPATH:{YOUR_REPO_PATH}` @@ -25,6 +36,7 @@ Test is performed in docker for this two platform. ``` ### Cambricon platform + 1. Please create a sitecustomize.py file in your python site-packages folder, and add the following code: ``` import site @@ -35,4 +47,4 @@ Test is performed in docker for this two platform. 4. All is ready! Let's test our models training in Ascend platform by the following command: ``` nohup bash run_train_all.sh >> output.log 2>&1 & - ``` + ``` diff --git a/Recommendation/DLRM/run_train.sh b/Recommendation/DLRM/run_train.sh index 69308da..ab3b5b9 100755 --- a/Recommendation/DLRM/run_train.sh +++ b/Recommendation/DLRM/run_train.sh @@ -13,7 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1 CUR_DIR=$(cd $(dirname $0); pwd) # 默认配置 -DATASET_DIR=${DATA_DIR:-""} # /data1/shared/Dataset/ml-20mx4x16 +DATASET_DIR=${DATA_DIR:-""} # ${BASE_DATASET_DIR}/ml-20mx4x16 THRESHOLD=${THRESHOLD:-1.0} # 默认阈值 ckp_dir=${CUR_DIR}/checkpoints # 检查点保存路径 cache_dir=${CUR_DIR}/data # 缓存目录 diff --git a/SR/ESPCN/data_utils.py b/SR/ESPCN/data_utils.py index d526a7c..16ac17e 100755 --- a/SR/ESPCN/data_utils.py +++ b/SR/ESPCN/data_utils.py @@ -59,7 +59,11 @@ def __len__(self): def generate_dataset(data_type, upscale_factor): - images_name = [x for x in listdir('../data/VOC2012-ESPCN/' + data_type) if is_image_file(x)] + base_dataset_dir = os.getenv('BASE_DATASET_DIR') + if base_dataset_dir is None: + raise ValueError("BASE_DATASET_DIR environment variable not set!") + data_path = os.path.join(base_dataset_dir, "VOC2012-ESPCN", data_type) + images_name = [x for x in listdir(data_path) if is_image_file(x)] crop_size = calculate_valid_crop_size(256, upscale_factor) lr_transform = input_transform(crop_size, upscale_factor) hr_transform = target_transform(crop_size) @@ -79,7 +83,7 @@ def generate_dataset(data_type, upscale_factor): for image_name in tqdm(images_name, desc='generate ' + data_type + ' dataset with upscale factor = ' + str(upscale_factor) + ' from VOC2012'): - image = Image.open('../data/VOC2012-ESPCN/' + data_type + '/' + image_name) + image = Image.open(os.path.join(data_path, image_name)) target = image.copy() image = lr_transform(image) target = hr_transform(target) diff --git a/SR/data/VOC2012-ESPCN b/SR/data/VOC2012-ESPCN deleted file mode 120000 index b9596e7..0000000 --- a/SR/data/VOC2012-ESPCN +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/VOC2012-ESPCN/ \ No newline at end of file diff --git a/Segmentation/data/VOCdevkit b/Segmentation/data/VOCdevkit index df9db1f..6113344 120000 --- a/Segmentation/data/VOCdevkit +++ b/Segmentation/data/VOCdevkit @@ -1 +1 @@ -/data1/shared/Dataset/VOCdevkit \ No newline at end of file +/data-aisoft/Dataset/VOCdevkit \ No newline at end of file diff --git a/Segmentation/deeplab/run_eval.sh b/Segmentation/deeplab/run_eval.sh index 6e4de9b..6e62e95 100755 --- a/Segmentation/deeplab/run_eval.sh +++ b/Segmentation/deeplab/run_eval.sh @@ -9,7 +9,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi python $SCRIPT_DIR/deeplab.py \ diff --git a/Segmentation/deeplab/run_train.sh b/Segmentation/deeplab/run_train.sh index 1d40eda..1b40e37 100755 --- a/Segmentation/deeplab/run_train.sh +++ b/Segmentation/deeplab/run_train.sh @@ -9,7 +9,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi python $SCRIPT_DIR/deeplab.py \ diff --git a/Segmentation/fcn/run_eval.sh b/Segmentation/fcn/run_eval.sh index a42ada8..88ed59b 100755 --- a/Segmentation/fcn/run_eval.sh +++ b/Segmentation/fcn/run_eval.sh @@ -9,7 +9,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi python $SCRIPT_DIR/fcn.py \ diff --git a/Segmentation/fcn/run_train.sh b/Segmentation/fcn/run_train.sh index 61d4fb9..afbc58a 100755 --- a/Segmentation/fcn/run_train.sh +++ b/Segmentation/fcn/run_train.sh @@ -9,7 +9,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi python $SCRIPT_DIR/fcn.py \ diff --git a/Segmentation/lraspp/run_eval.sh b/Segmentation/lraspp/run_eval.sh index f2f7b4a..6d94875 100755 --- a/Segmentation/lraspp/run_eval.sh +++ b/Segmentation/lraspp/run_eval.sh @@ -11,7 +11,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi # 参数解释 diff --git a/Segmentation/lraspp/run_train.sh b/Segmentation/lraspp/run_train.sh index 6adc2cb..1aaaa11 100755 --- a/Segmentation/lraspp/run_train.sh +++ b/Segmentation/lraspp/run_train.sh @@ -11,7 +11,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi # 参数解释 diff --git a/Segmentation/unet/run_eval.sh b/Segmentation/unet/run_eval.sh index 1ec3b5e..6115e59 100644 --- a/Segmentation/unet/run_eval.sh +++ b/Segmentation/unet/run_eval.sh @@ -12,7 +12,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi # 运行train.py diff --git a/Segmentation/unet/run_train.sh b/Segmentation/unet/run_train.sh index ecc1da6..f5ea934 100644 --- a/Segmentation/unet/run_train.sh +++ b/Segmentation/unet/run_train.sh @@ -11,7 +11,7 @@ export CUDA_VISIBLE_DEVICES=0 if [ -e "../data/VOCdevkit" ]; then echo "../data/VOCdevkit exists" else - ln -s /data1/shared/Dataset/VOCdevkit ../data/VOCdevkit + ln -s ${BASE_DATASET_DIR}/VOCdevkit ../data/VOCdevkit fi # 运行train.py diff --git a/Speech/data/LibriSpeech b/Speech/data/LibriSpeech index 582c366..7c4e54f 120000 --- a/Speech/data/LibriSpeech +++ b/Speech/data/LibriSpeech @@ -1 +1 @@ -/data1/shared/Dataset/librispeech/LibriSpeech/ \ No newline at end of file +/data-aisoft/Dataset/librispeech/LibriSpeech \ No newline at end of file diff --git a/Speech/data/data_thchs30 b/Speech/data/data_thchs30 index 231a6b6..5cc94f9 120000 --- a/Speech/data/data_thchs30 +++ b/Speech/data/data_thchs30 @@ -1 +1 @@ -/data1/shared/Dataset/data_thchs30 \ No newline at end of file +/data-aisoft/Dataset/data_thchs30 \ No newline at end of file diff --git a/Speech/deepspeech2/run_eval.sh b/Speech/deepspeech2/run_eval.sh index 15c4dd0..48d92b7 100644 --- a/Speech/deepspeech2/run_eval.sh +++ b/Speech/deepspeech2/run_eval.sh @@ -3,7 +3,7 @@ if [ -e "../data/data_thchs30" ]; then echo "../data/data_thchs30 exists" else - ln -s /data1/shared/Dataset/data_thchs30 ../data/data_thchs30 + ln -s ${BASE_DATASET_DIR}/data_thchs30 ../data/data_thchs30 fi if [ -e "./cache" ]; then diff --git a/Speech/deepspeech2/run_train.sh b/Speech/deepspeech2/run_train.sh index b3b7d10..4fe571f 100644 --- a/Speech/deepspeech2/run_train.sh +++ b/Speech/deepspeech2/run_train.sh @@ -3,7 +3,7 @@ if [ -e "../data/data_thchs30" ]; then echo "../data/data_thchs30 exists" else - ln -s /data1/shared/Dataset/data_thchs30 ../data_thchs30 + ln -s ${BASE_DATASET_DIR}/data_thchs30 ../data/data_thchs30 fi if [ -e "./cache" ]; then diff --git a/Speech/wav2vec/run_eval_offline.sh b/Speech/wav2vec/run_eval_offline.sh index 41f22ee..6557963 100755 --- a/Speech/wav2vec/run_eval_offline.sh +++ b/Speech/wav2vec/run_eval_offline.sh @@ -3,7 +3,7 @@ if [ -e "../data/LibriSpeech" ]; then echo "../data/LibriSpeech exists" else - ln -s /data1/shared/Dataset/librispeech/LibriSpeech ../data/LibriSpeech + ln -s ${BASE_DATASET_DIR}/librispeech/LibriSpeech ../data/LibriSpeech fi export CUDA_VISIBLE_DEVICES=0 diff --git a/Speech/wav2vec/run_eval_online.sh b/Speech/wav2vec/run_eval_online.sh index 27bbb86..e02e5b7 100755 --- a/Speech/wav2vec/run_eval_online.sh +++ b/Speech/wav2vec/run_eval_online.sh @@ -3,7 +3,7 @@ if [ -e "../data/LibriSpeech" ]; then echo "../data/LibriSpeech exists" else - ln -s /data1/shared/Dataset/librispeech/LibriSpeech ../data/LibriSpeech + ln -s ${BASE_DATASET_DIR}/librispeech/LibriSpeech ../data/LibriSpeech fi export CUDA_VISIBLE_DEVICES=0 diff --git a/Speech/wav2vec/run_train_offline.sh b/Speech/wav2vec/run_train_offline.sh index 2d70065..04caf1e 100755 --- a/Speech/wav2vec/run_train_offline.sh +++ b/Speech/wav2vec/run_train_offline.sh @@ -3,7 +3,7 @@ if [ -e "../data/LibriSpeech" ]; then echo "../data/LibriSpeech exists" else - ln -s /data1/shared/Dataset/librispeech/LibriSpeech ../data/LibriSpeech + ln -s ${BASE_DATASET_DIR}/librispeech/LibriSpeech ../data/LibriSpeech fi export CUDA_VISIBLE_DEVICES=0,1,2,3 diff --git a/Speech/wav2vec/run_train_online.sh b/Speech/wav2vec/run_train_online.sh index d353475..097e66e 100755 --- a/Speech/wav2vec/run_train_online.sh +++ b/Speech/wav2vec/run_train_online.sh @@ -3,7 +3,7 @@ if [ -e "../data/LibriSpeech" ]; then echo "../data/LibriSpeech exists" else - ln -s /data1/shared/Dataset/librispeech/LibriSpeech ../data/LibriSpeech + ln -s ${BASE_DATASET_DIR}/librispeech/LibriSpeech ../data/LibriSpeech fi export CUDA_VISIBLE_DEVICES=3,4,5,6 diff --git a/TimeSeriesPrediction/data/complete_data.csv b/TimeSeriesPrediction/data/complete_data.csv deleted file mode 120000 index a671a11..0000000 --- a/TimeSeriesPrediction/data/complete_data.csv +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/timeseq/complete_data.csv \ No newline at end of file diff --git a/TimeSeriesPrediction/data/mnist b/TimeSeriesPrediction/data/mnist deleted file mode 120000 index 3e80693..0000000 --- a/TimeSeriesPrediction/data/mnist +++ /dev/null @@ -1 +0,0 @@ -/data1/shared/Dataset/mnist/ \ No newline at end of file diff --git a/TimeSeriesPrediction/lstm/run_eval.sh b/TimeSeriesPrediction/lstm/run_eval.sh index 02ff35c..608a05f 100644 --- a/TimeSeriesPrediction/lstm/run_eval.sh +++ b/TimeSeriesPrediction/lstm/run_eval.sh @@ -17,4 +17,4 @@ python -W ignore eval.py \ --dataset $dataset \ --model_path $ckpt \ -# bash run_eval.sh ../data/complete_data.csv ./checkpoints/lstm_best.pt +# bash run_eval.sh ${BASE_DATASET_DIR}/timeseq/complete_data.csv ./checkpoints/lstm_best.pt diff --git a/TimeSeriesPrediction/lstm/run_train.sh b/TimeSeriesPrediction/lstm/run_train.sh index 699d378..bfe0fdf 100644 --- a/TimeSeriesPrediction/lstm/run_train.sh +++ b/TimeSeriesPrediction/lstm/run_train.sh @@ -15,4 +15,4 @@ python -W ignore train.py \ --batch_size $batch_size \ --lr $learning_rate \ -# bash run_train.sh ../data/complete_data.csv 200 512 0.0001 \ No newline at end of file +# bash run_train.sh ${BASE_DATASET_DIR}/timeseq/complete_data.csv 200 512 0.0001 \ No newline at end of file diff --git a/TimeSeriesPrediction/tcn/pmnist_test.py b/TimeSeriesPrediction/tcn/pmnist_test.py index b3ecd22..9cdbb3a 100644 --- a/TimeSeriesPrediction/tcn/pmnist_test.py +++ b/TimeSeriesPrediction/tcn/pmnist_test.py @@ -6,6 +6,7 @@ from model import TCN import numpy as np import argparse +import os import time from tqdm import tqdm @@ -43,7 +44,10 @@ if not args.cuda: print("WARNING: You have a CUDA device, so you should probably run with --cuda") -root = '../data/mnist' +base_dir = os.getenv('BASE_DATASET_DIR') +if base_dir is None: + raise ValueError("BASE_DATASET_DIR environment variable not set!") +root = os.path.join(base_dir, "mnist") batch_size = args.batch_size n_classes = 10 input_channels = 1 diff --git a/env.sh b/env.sh index 4935b2f..18d498c 100644 --- a/env.sh +++ b/env.sh @@ -1,3 +1,5 @@ # Choose from ["NVIDIA_GPU", "CAMBRICON_MLU", "ASCEND_NPU", # "METAX_GPU", "MOORE_GPU", "SUGON_DCU", "ILLUVATAR_GPU"] export PLATFORM_ENV="NVIDIA_GPU" + +export BASE_DATASET_DIR="/data-aisoft/Dataset" \ No newline at end of file diff --git a/run_train_all.sh b/run_train_all.sh index 5368617..786fe15 100644 --- a/run_train_all.sh +++ b/run_train_all.sh @@ -48,27 +48,27 @@ START_TIME=$(date +%s) run_test_fasterrcnn() { echo "-> 任务: Detection/fasterrcnn" - ( cd Detection/fasterrcnn && DATA_DIR=../data/VOCdevkit bash run_train.sh ) + ( cd Detection/fasterrcnn && DATA_DIR=${BASE_DATASET_DIR}/VOCdevkit bash run_train.sh ) } run_test_ssd() { echo "-> 任务: Detection/ssd" - ( cd Detection/ssd && DATA_DIR=../data/VOCdevkit bash run_train.sh ) + ( cd Detection/ssd && DATA_DIR=${BASE_DATASET_DIR}/VOCdevkit bash run_train.sh ) } run_test_yolo() { echo "-> 任务: Detection/yolo" - ( cd Detection/yolo && MODEL=yolov5s DATA_DIR=../data/coco bash run_train.sh ) + ( cd Detection/yolo && MODEL=yolov5s DATA_DIR=${BASE_DATASET_DIR}/coco bash run_train.sh ) } run_test_image_classification() { echo "-> 任务: ImageClassification/TorchVision" - ( cd ImageClassification/TorchVision && DATA_DIR=../data/imagenet2012 bash run_all_models_train.sh ) + ( cd ImageClassification/TorchVision && DATA_DIR=${BASE_DATASET_DIR}/imagenet2012 bash run_all_models_train.sh ) } run_test_gan() { echo "-> 任务: GAN/dcgan" - ( cd GAN/dcgan && DATA_DIR=../data/lsun bash run_train.sh ) + ( cd GAN/dcgan && DATA_DIR=${BASE_DATASET_DIR}/lsun bash run_train.sh ) } run_test_nlp() { @@ -83,7 +83,7 @@ run_test_rl() { run_test_recommendation() { echo "-> 任务: Recommendation/DLRM" - ( cd Recommendation/DLRM && DATA_DIR=../data/ml-20mx4x16 bash run_train.sh ) + ( cd Recommendation/DLRM && DATA_DIR=${BASE_DATASET_DIR}/ml-20mx4x16 bash run_train.sh ) } run_test_sr() { @@ -123,7 +123,7 @@ run_test_speech_wav2vec() { run_test_timeseries_lstm() { echo "-> 任务: TimeSeriesPrediction/lstm" - ( cd TimeSeriesPrediction/lstm && bash run_train.sh ../data/complete_data.csv 200 512 0.0001 ) + ( cd TimeSeriesPrediction/lstm && bash run_train.sh ${BASE_DATASET_DIR}/timeseq/complete_data.csv 200 512 0.0001 ) } run_test_timeseries_tcn() {