Updated script to better work with slurm #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

knoriy wants to merge 47 commits into LAION-AI:main from knoriy:main

.gitignore

-Original file line number
+Diff line change
@@ Expand Up / @@ -133,4 +133,5 @@ dmypy.json @@
     utils/__pycache__/make_tar_utils.cpython-37.pyc
     /data_preprocess/process_audioset/
-    *.out
+    *.out
+    test.*py*

current_dataset/ToDO.md

-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+    # ToDo
+    - [X] LJSpeech
+    - [X] MSWC
+    - [ ] GigaSpeech
+    - [ ] CoVoST

current_dataset/preprocess_CREMA-D.py

-Original file line number
+Diff line change
@@ -0,0 +1,130 @@
+    """
+    Code for preprocess GigaSpeech Corpus:
+    https://github.com/SpeechColab/GigaSpeech
+    """
+    import glob
+    import tqdm
+    import os
+    import glob
+    import pandas as pd
+    import sys
+    import tarfile
+    import json
+    import shutil
+    import fsspec
+    from sklearn.model_selection import train_test_split
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    from multiprocessing import Pool
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+    from utils.audio_utils import audio_to_flac
+    from utils.make_tar_utils import tardir
+    def convert_and_json_dump(file:str, dest:str, df, overwrite:bool=True, verbose=False):
+        if os.path.isfile(dest) and overwrite==False:
+            if verbose==True:
+                print(f'{dest} already exists, skiping')
+            return
+        audio_to_flac(file, dest)
+        with open(dest.replace('.flac', '.json'), 'w') as f:
+            json.dump({'filename': os.path.join(*dest.split('/')[5:]), 'text':[df['text']], 'original_data':df['tag']}, f)
+    def split_all_audio_files(df, dest_root_path, max_workers=96):
+        if not os.path.exists(dest_root_path):
+            raise FileNotFoundError(f'Please Check {dest_root_path} exists')
+        l = len(df)
+        with tqdm.tqdm(total=l, desc=f'Processing {dest_root_path}') as pbar:
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                threads = [executor.submit(convert_and_json_dump, row[0], os.path.join(dest_root_path, f'{i}.flac'), row) for i, row in enumerate(df.iloc())]
+                for _ in as_completed(threads):
+                    pbar.update(1)
+    def create_df(root_path:str, dataset_name:str=None):
+        wavs = glob.glob(os.path.join(root_path, 'AudioWAV',  '**/*.wav'), recursive=True)
+        codes = {   'Statement':{   'IEO':"It's eleven o'clock",
+                                    'TIE':"That is exactly what happened",
+                                    'IOM':"I'm on my way to the meeting",
+                                    'IWW':"I wonder what this is about",
+                                    'TAI':"The airplane is almost full",
+                                    'MTI':"Maybe tomorrow it will be cold",
+                                    'IWL':"I would like a new alarm clock",
+                                    'ITH':"I think I have a doctor's appointment",
+                                    'DFA':"Don't forget a jacket",
+                                    'ITS':"I think I've seen this before",
+                                    'TSI':"The surface is slick",
+                                    'WSI':"We'll stop in a couple of minutes",
+                                    },
+                    'Emotion':{     'ANG':'angry',
+                                    'DIS':'disgusted',
+                                    'FEA':'fearful',
+                                    'HAP':'happy',
+                                    'NEU':'neutral',
+                                    'SAD':'sad',
+                            },
+                    'Emotional intensity':{ 'LO':'low',
+                                            'MD':'medium',
+                                            'HI':'high',
+                                            'XX':'unspecified',
+                                            },
+                    }
+        demographics = pd.read_csv(os.path.join(root_path, 'VideoDemographics.csv'), names=["ActorID","Age","Sex","Race","Ethnicity"])
+        df_data = []
+        for wav in tqdm.tqdm(wavs):
+            file_name = os.path.basename(wav).split('.')[0]
+            wav_codes = file_name.split('_')
+            text_meta = [codes['Statement'][wav_codes[1]], codes['Emotion'][wav_codes[2]], codes['Emotional intensity'][wav_codes[3]]]
+            demograpthics_meta = demographics.loc[demographics['ActorID'] == wav_codes[0]]
+            male_or_female = 'woman' if demograpthics_meta["Sex"].values[0] == 'Female' else 'man'
+            intensity = '' if text_meta[2] == 'unspecified' else f' with {text_meta[2]} emotional intensity'
+            text = f'A {demograpthics_meta["Age"].values[0]} year-old {male_or_female}, saying "{text_meta[0]}" in a {text_meta[1]} voice{intensity}.'
+            df_data.append({ 'path':wav, 'text':text, 'tag':{'transcript':text_meta[0], 'language':'english', 'emotion':text_meta[1], 'emotion_intensity':text_meta[2], 'gender':demograpthics_meta["Sex"].values[0], 'age':demograpthics_meta["Age"].values[0] }})
+        return pd.DataFrame(df_data)
+    if __name__ == '__main__':
+        import multiprocessing
+        max_workers = multiprocessing.cpu_count()
+        print("Num workers: ", max_workers)
+        chunk = 512
+        root_path = '/admin/home-knoriy/DELETEME/CREMA-D/'
+        dataset_name = 'CREMA-D'
+        s3 = fsspec.filesystem('s3')
+        s3_dest = f'laion-west-audio/webdataset_tar/{dataset_name}/'
+        original_tar_dir = '/fsx/knoriy/raw_datasets/CREMA-D/crema-d.tar.gz'
+        # print('Extracting tar')
+        # with tarfile.open(original_tar_dir, mode='r:gz') as file:
+        #     audio_path = os.path.split(original_tar_dir)[0]
+        #     file.extractall(audio_path)
+        # load metadata and configure audio paths
+        df = create_df(root_path)
+        # create train, test, valid splits
+        train, test = train_test_split(df, test_size=0.2)
+        valid, test = train_test_split(test, test_size=0.2)
+        train_test_val = {'valid/':valid, 'train/':train, 'test/':test}
+        for key in tqdm.tqdm(train_test_val, desc=f'processing:'):
+            df = train_test_val[key]
+            dest_path = os.path.join(root_path.replace('CREMA-D', 'CREMA-D_processed').replace('AudioWAV/', ''), key)
+            os.makedirs(dest_path, exist_ok=True)
+            split_all_audio_files(df, dest_path)
+            tardir(dest_path, dest_path, chunk, delete_file=True)
+            # upload to s3 and delete local
+            s3.put(dest_path, os.path.join(s3_dest, key), recursive=True)
+            shutil.rmtree(dest_path)

current_dataset/preprocess_CoVoST.py

-Original file line number
+Diff line change
@@ -0,0 +1,192 @@
+    """
+    Code for preprocess LJSpeech Corpus:
+    https://keithito.com/LJ-Speech-Dataset/
+    """
+    import glob
+    from tabnanny import verbose
+    from tokenize import Name
+    import tqdm
+    import os
+    import glob
+    import pandas as pd
+    import sys
+    import tarfile
+    import json
+    import shutil
+    import fsspec
+    from sklearn.model_selection import train_test_split
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+    from utils.audio_utils import audio_to_flac
+    from utils.make_tar_utils import tardir
+    def convert_and_json_dump(file:str, dest:str, df, overwrite:bool=False, verbose=False):
+        if os.path.isfile(dest) and overwrite==False:
+            if verbose==True:
+                print(f'{dest} already exists, skiping')
+            return
+        audio_to_flac(file, dest)
+        with open(dest.replace('.flac', '.json'), 'w') as f:
+            json.dump({'filename': os.path.join(*dest.split('/')[3:]), 'text':[df['text']], 'original_data':df['original_data']}, f)
+    def split_all_audio_files(df, dest_root_path, max_workers=96):
+        if not os.path.exists(dest_root_path):
+            raise FileNotFoundError(f'Please Check {dest_root_path} exists')
+        l = len(df)
+        with tqdm.tqdm(total=l, desc=f'Processing {dest_root_path}') as pbar:
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                threads = [executor.submit(convert_and_json_dump, row[0], os.path.join(dest_root_path, f'{i}.flac'), row, overwrite=False, verbose=False) for i, row in enumerate(df.iloc())]
+                for _ in as_completed(threads):
+                    pbar.update(1)
+    def download_tsvs(urls:list, output_dir:str, extract:bool=False):
+        os.makedirs(output_dir, exist_ok=True)
+        for url in tqdm.tqdm(urls, desc="Downloading tsvs"):
+            dest_path = os.path.join(output_dir, url.split("/")[-1])
+            if os.path.isfile(dest_path):
+                continue
+            os.system(f'curl {url} --output {dest_path}')
+            if extract:
+                os.system(f'tar -xf {dest_path}')
+    def extract_covost_2_tsvs(tsv_tar_dir:str, dest:str, cv_tsv:str, version=2):
+        extract_covost_2_tar_cmd = f'tar -xf {tsv_tar_dir} -C {dest}'
+        os.system(extract_covost_2_tar_cmd)
+        src_lang, tgt_lang = os.path.basename(tsv_tar_dir).split('.')[1].split('_')
+        get_covost_splits_cmd = f'python /home/knoriy/fsx/raw_datasets/CoVoST_2/covost/get_covost_splits.py \
+            --version {version} \
+            --src-lang {src_lang} \
+            --tgt-lang {tgt_lang} \
+            --root {dest} \
+            --cv-tsv {cv_tsv} \
+            '
+        os.system(get_covost_splits_cmd)
+    if __name__ == '__main__':
+        import multiprocessing
+        x_2_eng = [
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.fr_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.de_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.es_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.ca_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.it_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.ru_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.zh-CN_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.pt_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.fa_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.et_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.mn_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.nl_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.tr_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.ar_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.sv-SE_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.lv_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.sl_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.ta_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.ja_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.id_en.tsv.tar.gz",
+            "https://dl.fbaipublicfiles.com/covost/covost_v2.cy_en.tsv.tar.gz",
+        ]
+        eng_2_x = [
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_de.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_ca.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_zh-CN.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_fa.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_et.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_mn.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_tr.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_ar.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_sv-SE.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_lv.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_sl.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_ta.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_ja.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_id.tsv.tar.gz',
+            'https://dl.fbaipublicfiles.com/covost/covost_v2.en_cy.tsv.tar.gz',
+        ]
+        get_language_from_key = {
+            'en':'english',
+            'de':'german',
+            'fr':'french',
+            'nl':'dutch',
+            'ru':'russian',
+            'es':'spanish',
+            'it':'italian',
+            'tr':'turkish',
+            'fa':'persian',
+            'ca':'catalan',
+            'zh-cn':'chinese',
+            'pt':'portuguese',
+            'et':'estonian',
+            'mn':'mongolian',
+            'ar':'arabic',
+            'sv-se':'swedish',
+            'lv':'latvian',
+            'sl':'slovenian',
+            'ta':'tamil',
+            'ja':'japanese',
+            'id':'indonesian',
+            'cy':'welsh',
+            }
+        max_workers = multiprocessing.cpu_count()
+        chunk = 512
+        generate_subset_tsv = True
+        root_path = '/home/knoriy/fsx/raw_datasets/CoVoST_2/'
+        metadata_dir = "/home/knoriy/fsx/raw_datasets/CoVoST_2/"
+        dataset_name = 'CoVoST_2'
+        COMMON_VOICE_VERSION = 'cv-corpus-10.0-2022-07-04'
+        s3 = fsspec.filesystem('s3')
+        s3_dest = f's-laion/knoriy/{dataset_name}/{dataset_name}_tars/'
+        download_tsvs(eng_2_x, os.path.join(root_path, 'tsvs/'))
+        # download_tsvs(x_2_eng, "/home/knoriy/fsx/raw_datasets/CoVoST_2/tsvs")
+        # uncomment to extract and create CoVoST tsvs
+        # for tar in tqdm.tqdm(glob.glob(os.path.join(root_path, 'tsvs/**/*.tar.gz'), recursive=True), desc='Extracting tsvs'):
+        #     extract_covost_2_tsvs(tar, os.path.join(root_path, 'tsvs/'), '/home/knoriy/fsx/raw_datasets/CoVoST_2/cv-corpus-10.0-2022-07-04/en/validated.tsv')
+        # load metadata and configure audio paths
+        tsvs = []
+        for tsv in glob.glob(os.path.join(root_path, 'tsvs/**/*.tsv'), recursive=True):
+            if any(word in os.path.basename(tsv) for word in ['test', 'train', 'dev']):
+                tsvs.append(tsv)
+        for tsv in tqdm.tqdm(tsvs, desc=f'processing:'):
+            raw_df = pd.read_csv(tsv, sep='\t', on_bad_lines='skip')
+            IS_TRAIN_VAL_OR_TEST, LANGUAGE = tsv.split('.')[-2], tsv.split('.')[-3]
+            data = {}
+            for row in raw_df.iloc():
+                data.setdefault('paths', []).append(os.path.join(root_path, COMMON_VOICE_VERSION, LANGUAGE.split('_')[0], "clips", row['path']))
+                data.setdefault('text', []).append(f"{row['translation']} translated to {get_language_from_key[LANGUAGE.split('_')[0]]}")
+                data.setdefault('original_data', []).append(
+                    {
+                        "sentence":row['sentence'],
+                        "translation":row['translation'],
+                        "client_id":row['client_id'],
+                    }
+                )
+            df = pd.DataFrame(data)
+            dest_path = os.path.join(root_path.replace('raw_datasets', 'processed_datasets'), LANGUAGE, IS_TRAIN_VAL_OR_TEST, '')
+            os.makedirs(dest_path, exist_ok=True)
+            split_all_audio_files(df, dest_path)
+            tardir(dest_path, dest_path, chunk, delete_file=True)
+            # upload to s3 and delete local
+            s3.put(dest_path, os.path.join(s3_dest, LANGUAGE, IS_TRAIN_VAL_OR_TEST)+'/', recursive=True)
+            shutil.rmtree(dest_path)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Updated script to better work with slurm #39

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Updated script to better work with slurm #39

Are you sure you want to change the base?

Uh oh!

Updated script to better work with slurm #39

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!