rm upstream example

docs/example-metadata.py

@@ -1,63 +0,0 @@
-import datasets
-from pathlib import Path
-from tqdm.auto import tqdm
-import pandas as pd
-
-# assuming metadata is cloned into the local folder /data/hf_repos/the-stack-metadata
-#          the stack is cloned into the local folder /data/hf_repos/the-stack-v1.1
-#          destination folder is in  /repo_workdir/numpy_restored
-the_stack_meta_path = Path('/srv/ml/huggingface/datasets/bigcode/the-stack-metadata')
-the_stack_path = Path('/data/hf_repos/the-stack-v1.1')
-repo_dst_root = Path('/repo_workdir/numpy_restored')
-repo_name = 'numpy/numpy'
-
-# Get bucket with numpy repo info
-# meta_bucket_path = None
-#for fn in tqdm(list((the_stack_meta_path/'data').glob('*/ri.parquet'))):
-#    df = pd.read_parquet(fn)
-#    if any(df['name'] == repo_name):
-#        meta_bucket_path = fn
-#        break
-meta_bucket_path = the_stack_meta_path / 'data/255_944'
-
-
-# Get repository id from repo name
-ri_id = pd.read_parquet(
-    meta_bucket_path / 'ri.parquet'
-).query(
-    f'`name` == "{repo_name}"'
-)['id'].to_list()[0]
-
-# Get files information for the reopository
-files_info = pd.read_parquet(
-    meta_bucket_path / 'fi.parquet'
-).query(
-    f'`ri_id` == {ri_id} and `size` != 0 and `is_deleted` == False'
-)
-
-# Convert DF with files information to a dictionary by language and then file hexsha
-#   there can be more than one file with the same  hexsha in the repo so we gather
-#   all instances per unique hexsha
-files_info_dict = {
-    k: v[['hexsha', 'path']].groupby('hexsha').apply(lambda x: list(x['path'])).to_dict()
-    for k, v in files_info.groupby('lang_ex')
-}
-
-# Load Python part of The Stack
-ds = datasets.load_dataset(
-    str(the_stack_path/'data/python'),
-    num_proc=10, ignore_verifications=True
-)
-
-# Save file content of the python files in the numpy reposirotry in their appropriate locations
-def save_file_content(example, files_info_dict, repo_dst_root):
-    if example['hexsha'] in  files_info_dict:
-        for el in files_info_dict[example['hexsha']]:
-            path = repo_dst_root / el
-            path.parent.mkdir(parents=True, exist_ok=True)
-            path.write_text(example['content'])
-ds.map(
-    save_file_content,
-    fn_kwargs={'files_info_dict': files_info_dict['Python'], 'repo_dst_root': repo_dst_root},
-    num_proc=10
-)