[ready] Replacing os with pathlib (#1708)
* replace os.path with pathlib * safe convert dirnames to pathlib * replace all os.path.join * fix cuda error * change main chunk * Reviewer fixes * fix vgg * Fixed everything * Final fixes * ensure consistency * Change all parent.parent... to parentspull/1718/head
Karan Handa
GitHub
parent
355b02dc3f
commit
a8aa13dc91
|
|
@ -8,7 +8,7 @@ def disasm(buf):
|
|||
global fxn
|
||||
if fxn is None:
|
||||
shared = pathlib.Path(__file__).parent / "disasm.so"
|
||||
if not os.path.isfile(shared):
|
||||
if not shared.is_file():
|
||||
os.system(f'cd {pathlib.Path(__file__).parent} && gcc -shared disasm-a3xx.c -o disasm.so')
|
||||
fxn = ctypes.CDLL(shared.as_posix())['disasm']
|
||||
#hexdump(buf)
|
||||
|
|
|
|||
|
|
@ -1,20 +1,22 @@
|
|||
from pathlib import Path
|
||||
from models.efficientnet import EfficientNet
|
||||
from tinygrad.tensor import Tensor
|
||||
from tinygrad.nn.state import safe_save
|
||||
from extra.utils import fetch
|
||||
from extra.export_model import export_model
|
||||
from tinygrad.helpers import getenv
|
||||
import ast, os
|
||||
import ast
|
||||
|
||||
if __name__ == "__main__":
|
||||
model = EfficientNet(0)
|
||||
model.load_from_pretrained()
|
||||
mode = "clang" if getenv("CLANG", "") != "" else "webgpu" if getenv("WEBGPU", "") != "" else ""
|
||||
prg, inp_size, out_size, state = export_model(model, Tensor.randn(1,3,224,224), mode)
|
||||
dirname = Path(__file__).parent
|
||||
if getenv("CLANG", "") == "":
|
||||
safe_save(state, os.path.join(os.path.dirname(__file__), "net.safetensors"))
|
||||
safe_save(state, (dirname / "net.safetensors").as_posix())
|
||||
ext = "js" if getenv("WEBGPU", "") != "" else "json"
|
||||
with open(os.path.join(os.path.dirname(__file__), f"net.{ext}"), "w") as text_file:
|
||||
with open(dirname / f"net.{ext}", "w") as text_file:
|
||||
text_file.write(prg)
|
||||
else:
|
||||
cprog = [prg]
|
||||
|
|
|
|||
|
|
@ -227,7 +227,7 @@ def concat_weights(models):
|
|||
def load(fn:str):
|
||||
if fn.endswith('.index.json'):
|
||||
with open(fn) as fp: weight_map = json.load(fp)['weight_map']
|
||||
parts = {n: load(f'{os.path.dirname(fn)}/{os.path.basename(n)}') for n in set(weight_map.values())}
|
||||
parts = {n: load(Path(fn).parent / Path(n).name) for n in set(weight_map.values())}
|
||||
return {k: parts[n][k] for k, n in weight_map.items()}
|
||||
elif fn.endswith('.safetensors'):
|
||||
return safe_load(fn)
|
||||
|
|
@ -428,7 +428,7 @@ After you are done speaking, output [EOS]. You are not Chad.
|
|||
|
||||
|
||||
LLAMA_SUFFIX = {1: "", 2: "-2"}[args.gen]
|
||||
MODEL_PATH = args.model or Path(__file__).parent.parent / f"weights/LLaMA{LLAMA_SUFFIX}/{args.size}"
|
||||
MODEL_PATH = args.model or Path(__file__).parents[1] / f"weights/LLaMA{LLAMA_SUFFIX}/{args.size}"
|
||||
TOKENIZER_PATH = (MODEL_PATH if MODEL_PATH.is_dir() else MODEL_PATH.parent) / "tokenizer.model"
|
||||
print(f"using LLaMA{LLAMA_SUFFIX}-{args.size} model")
|
||||
llama = LLaMa.build(MODEL_PATH, TOKENIZER_PATH, model_gen=args.gen, model_size=args.size, quantize=args.quantize)
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ def eval_bert():
|
|||
from examples.mlperf.metrics import f1_score
|
||||
from transformers import BertTokenizer
|
||||
|
||||
tokenizer = BertTokenizer(str(Path(__file__).parent.parent.parent / "weights/bert_vocab.txt"))
|
||||
tokenizer = BertTokenizer(str(Path(__file__).parents[2] / "weights/bert_vocab.txt"))
|
||||
|
||||
c = 0
|
||||
f1 = 0.0
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# original implementation: https://github.com/svc-develop-team/so-vits-svc
|
||||
from __future__ import annotations
|
||||
import sys, os, logging, time, io, math, argparse, operator, numpy as np
|
||||
import sys, logging, time, io, math, argparse, operator, numpy as np
|
||||
from functools import partial, reduce
|
||||
from pathlib import Path
|
||||
from typing import Tuple, Optional, Type
|
||||
|
|
@ -468,14 +468,14 @@ def repeat_expand_2d_left(content, target_len): # content : [h, t]
|
|||
return Tensor.stack(cols).transpose(0, 1)
|
||||
|
||||
def load_fairseq_cfg(checkpoint_path):
|
||||
assert os.path.isfile(checkpoint_path)
|
||||
assert Path(checkpoint_path).is_file()
|
||||
state = torch_load(checkpoint_path)
|
||||
cfg = state["cfg"] if ("cfg" in state and state["cfg"] is not None) else None
|
||||
if cfg is None: raise RuntimeError(f"No cfg exist in state keys = {state.keys()}")
|
||||
return HParams(**cfg)
|
||||
|
||||
def load_checkpoint_enc(checkpoint_path, model: ContentVec, optimizer=None, skip_list=[]):
|
||||
assert os.path.isfile(checkpoint_path)
|
||||
assert Path(checkpoint_path).is_file()
|
||||
start_time = time.time()
|
||||
checkpoint_dict = torch_load(checkpoint_path)
|
||||
saved_state_dict = checkpoint_dict['model']
|
||||
|
|
@ -550,7 +550,7 @@ def get_encoder(ssl_dim) -> Type[SpeechEncoder]:
|
|||
# DEMO USAGE (uses audio sample from LJ-Speech):
|
||||
# python3 examples/so_vits_svc.py --model saul_goodman
|
||||
#########################################################################################
|
||||
SO_VITS_SVC_PATH = Path(__file__).parent.parent / "weights/So-VITS-SVC"
|
||||
SO_VITS_SVC_PATH = Path(__file__).parents[1] / "weights/So-VITS-SVC"
|
||||
VITS_MODELS = { # config_path, weights_path, config_url, weights_url
|
||||
"saul_goodman" : (SO_VITS_SVC_PATH / "config_saul_gman.json", SO_VITS_SVC_PATH / "pretrained_saul_gman.pth", "https://huggingface.co/Amo/so-vits-svc-4.0_GA/resolve/main/ModelsFolder/Saul_Goodman_80000/config.json", "https://huggingface.co/Amo/so-vits-svc-4.0_GA/resolve/main/ModelsFolder/Saul_Goodman_80000/G_80000.pth"),
|
||||
"drake" : (SO_VITS_SVC_PATH / "config_drake.json", SO_VITS_SVC_PATH / "pretrained_drake.pth", "https://huggingface.co/jaspa/so-vits-svc/resolve/main/aubrey/config_aubrey.json", "https://huggingface.co/jaspa/so-vits-svc/resolve/main/aubrey/pretrained_aubrey.pth"),
|
||||
|
|
@ -563,13 +563,13 @@ ENCODER_MODELS = { # weights_path, weights_url
|
|||
"contentvec": (SO_VITS_SVC_PATH / "contentvec_checkpoint.pt", "https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt")
|
||||
}
|
||||
ENCODER_MODEL = "contentvec"
|
||||
DEMO_PATH, DEMO_URL = Path(__file__).parent.parent / "temp/LJ037-0171.wav", "https://keithito.com/LJ-Speech-Dataset/LJ037-0171.wav"
|
||||
DEMO_PATH, DEMO_URL = Path(__file__).parents[1] / "temp/LJ037-0171.wav", "https://keithito.com/LJ-Speech-Dataset/LJ037-0171.wav"
|
||||
if __name__=="__main__":
|
||||
logging.basicConfig(stream=sys.stdout, level=(logging.INFO if DEBUG < 1 else logging.DEBUG))
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-m", "--model", default=None, help=f"Specify the model to use. All supported models: {VITS_MODELS.keys()}", required=True)
|
||||
parser.add_argument("-f", "--file", default=DEMO_PATH, help=f"Specify the path of the input file")
|
||||
parser.add_argument("--out_dir", default=str(Path(__file__).parent.parent / "temp"), help="Specify the output path.")
|
||||
parser.add_argument("--out_dir", default=str(Path(__file__).parents[1] / "temp"), help="Specify the output path.")
|
||||
parser.add_argument("--out_path", default=None, help="Specify the full output path. Overrides the --out_dir and --name parameter.")
|
||||
parser.add_argument("--base_name", default="test", help="Specify the base of the output file name. Default is 'test'.")
|
||||
parser.add_argument("--speaker", default=None, help="If not specified, the first available speaker is chosen. Usually there is only one speaker per model.")
|
||||
|
|
@ -600,7 +600,7 @@ if __name__=="__main__":
|
|||
|
||||
### Loading audio and slicing ###
|
||||
if audio_path == DEMO_PATH: download_if_not_present(DEMO_PATH, DEMO_URL)
|
||||
assert os.path.isfile(audio_path) and Path(audio_path).suffix == ".wav"
|
||||
assert Path(audio_path).is_file() and Path(audio_path).suffix == ".wav"
|
||||
chunks = preprocess.cut(audio_path, db_thresh=slice_db)
|
||||
audio_data, audio_sr = preprocess.chunks2audio(audio_path, chunks)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
# https://arxiv.org/pdf/2112.10752.pdf
|
||||
# https://github.com/ekagra-ranjan/huggingface-blog/blob/main/stable_diffusion.md
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import gzip, argparse, math, re
|
||||
|
|
@ -424,7 +423,7 @@ class CLIPTextTransformer:
|
|||
# Clip tokenizer, taken from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py (MIT license)
|
||||
@lru_cache()
|
||||
def default_bpe():
|
||||
fn = Path(__file__).parent.parent / "weights/bpe_simple_vocab_16e6.txt.gz"
|
||||
fn = Path(__file__).parents[1] / "weights/bpe_simple_vocab_16e6.txt.gz"
|
||||
download_file("https://github.com/openai/CLIP/raw/main/clip/bpe_simple_vocab_16e6.txt.gz", fn)
|
||||
return fn
|
||||
|
||||
|
|
@ -558,13 +557,13 @@ class StableDiffusion:
|
|||
# cond_stage_model.transformer.text_model
|
||||
|
||||
# this is sd-v1-4.ckpt
|
||||
FILENAME = Path(__file__).parent.parent / "weights/sd-v1-4.ckpt"
|
||||
FILENAME = Path(__file__).parents[1] / "weights/sd-v1-4.ckpt"
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Run Stable Diffusion', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--steps', type=int, default=5, help="Number of steps in diffusion")
|
||||
parser.add_argument('--prompt', type=str, default="a horse sized cat eating a bagel", help="Phrase to render")
|
||||
parser.add_argument('--out', type=str, default=os.path.join(tempfile.gettempdir(), "rendered.png"), help="Output filename")
|
||||
parser.add_argument('--out', type=str, default=Path(tempfile.gettempdir()) / "rendered.png", help="Output filename")
|
||||
parser.add_argument('--noshow', action='store_true', help="Don't show the image")
|
||||
parser.add_argument('--fp16', action='store_true', help="Cast the weights to float16")
|
||||
parser.add_argument('--timing', action='store_true', help="Print timing per step")
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import sys
|
||||
import os
|
||||
import random
|
||||
import json
|
||||
import numpy
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tinygrad.tensor import Tensor
|
||||
from tinygrad.nn.optim import SGD
|
||||
|
|
@ -80,8 +80,7 @@ if cmd == "import":
|
|||
|
||||
vgg7.load_waifu2x_json(json.load(open(src, "rb")))
|
||||
|
||||
if not os.path.isdir(model):
|
||||
os.mkdir(model)
|
||||
Path(model).mkdir(exist_ok=True)
|
||||
load_and_save(model, True)
|
||||
elif cmd == "execute":
|
||||
model = sys.argv[2]
|
||||
|
|
@ -102,8 +101,7 @@ elif cmd == "execute_full":
|
|||
elif cmd == "new":
|
||||
model = sys.argv[2]
|
||||
|
||||
if not os.path.isdir(model):
|
||||
os.mkdir(model)
|
||||
Path(model).mkdir(exist_ok=True)
|
||||
load_and_save(model, True)
|
||||
elif cmd == "train":
|
||||
model = sys.argv[2]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from tinygrad.tensor import Tensor
|
||||
import numpy
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Format Details:
|
||||
# A KINNE parameter set is stored as a set of files named "snoop_bin_*.bin",
|
||||
|
|
@ -35,8 +35,8 @@ class KinneDir:
|
|||
It is important that if you wish to save in the current directory,
|
||||
you use ".", not the empty string.
|
||||
"""
|
||||
if save and not os.path.isdir(base):
|
||||
os.mkdir(base)
|
||||
if save:
|
||||
Path(base).mkdir(exist_ok=True)
|
||||
self.base = base + "/snoop_bin_"
|
||||
self.next_part_index = 0
|
||||
self.save = save
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import json, logging, math, os, re, sys, time, wave, argparse, numpy as np
|
||||
import json, logging, math, re, sys, time, wave, argparse, numpy as np
|
||||
from functools import reduce
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
|
@ -522,7 +522,7 @@ def load_model(symbols, hps, model) -> Synthesizer:
|
|||
_ = load_checkpoint(weights_path, net_g, None)
|
||||
return net_g
|
||||
def load_checkpoint(checkpoint_path, model: Synthesizer, optimizer=None, skip_list=[]):
|
||||
assert os.path.isfile(checkpoint_path)
|
||||
assert Path(checkpoint_path).is_file()
|
||||
start_time = time.time()
|
||||
checkpoint_dict = torch_load(checkpoint_path)
|
||||
iteration, learning_rate = checkpoint_dict['iteration'], checkpoint_dict['learning_rate']
|
||||
|
|
@ -556,8 +556,8 @@ def load_checkpoint(checkpoint_path, model: Synthesizer, optimizer=None, skip_li
|
|||
return model, optimizer, learning_rate, iteration
|
||||
|
||||
def download_if_not_present(file_path: Path, url: str):
|
||||
if not os.path.isfile(file_path):
|
||||
logging.info(f"Did not find {file_path}, downloading...")
|
||||
if not file_path.is_file():
|
||||
logging.info(f"Did not find {file_path.as_posix()}, downloading...")
|
||||
download_file(url, file_path)
|
||||
return file_path
|
||||
|
||||
|
|
@ -649,7 +649,7 @@ class TextMapper: # Based on https://github.com/keithito/tacotron
|
|||
# anime lady 1 | --model_to_use uma_trilingual --speaker_id 36
|
||||
# anime lady 2 | --model_to_use uma_trilingual --speaker_id 121
|
||||
#########################################################################################
|
||||
VITS_PATH = Path(__file__).parent.parent / "weights/VITS/"
|
||||
VITS_PATH = Path(__file__).parents[1] / "weights/VITS/"
|
||||
MODELS = { # config_path, weights_path, config_url, weights_url
|
||||
"ljs": (VITS_PATH / "config_ljs.json", VITS_PATH / "pretrained_ljs.pth", "https://raw.githubusercontent.com/jaywalnut310/vits/main/configs/ljs_base.json", "https://drive.google.com/uc?export=download&id=1q86w74Ygw2hNzYP9cWkeClGT5X25PvBT&confirm=t"),
|
||||
"vctk": (VITS_PATH / "config_vctk.json", VITS_PATH / "pretrained_vctk.pth", "https://raw.githubusercontent.com/jaywalnut310/vits/main/configs/vctk_base.json", "https://drive.google.com/uc?export=download&id=11aHOlhnxzjpdWDpsz1vFDCzbeEfoIxru&confirm=t"),
|
||||
|
|
@ -665,7 +665,7 @@ if __name__ == '__main__':
|
|||
parser.add_argument("--model_to_use", default="vctk", help="Specify the model to use. Default is 'vctk'.")
|
||||
parser.add_argument("--speaker_id", type=int, default=6, help="Specify the speaker ID. Default is 6.")
|
||||
parser.add_argument("--out_path", default=None, help="Specify the full output path. Overrides the --out_dir and --name parameter.")
|
||||
parser.add_argument("--out_dir", default=str(Path(__file__).parent.parent / "temp"), help="Specify the output path.")
|
||||
parser.add_argument("--out_dir", default=str(Path(__file__).parents[1] / "temp"), help="Specify the output path.")
|
||||
parser.add_argument("--base_name", default="test", help="Specify the base of the output file name. Default is 'test'.")
|
||||
parser.add_argument("--text_to_synthesize", default="""Hello person. If the code you are contributing isn't some of the highest quality code you've written in your life, either put in the effort to make it great, or don't bother.""", help="Specify the text to synthesize. Default is a greeting message.")
|
||||
parser.add_argument("--noise_scale", type=float, default=0.667, help="Specify the noise scale. Default is 0.667.")
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ LANGUAGES = {
|
|||
"as": "assamese", "tt": "tatar", "haw": "hawaiian", "ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese",
|
||||
}
|
||||
|
||||
BASE = pathlib.Path(__file__).parent.parent / "weights"
|
||||
BASE = pathlib.Path(__file__).parents[1] / "weights"
|
||||
def get_encoding(n_vocab_in):
|
||||
download_file("https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/gpt2.tiktoken", BASE / "gpt2.tiktoken")
|
||||
ranks = {base64.b64decode(token): int(rank) for token, rank in (line.split() for line in open(BASE / "gpt2.tiktoken") if line)}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
import os
|
||||
from ultralytics import YOLO
|
||||
import onnx
|
||||
from pathlib import Path
|
||||
from extra.onnx import get_run_onnx
|
||||
from tinygrad.tensor import Tensor
|
||||
|
||||
os.chdir("/tmp")
|
||||
if not os.path.isfile("yolov8n-seg.onnx"):
|
||||
if not Path("yolov8n-seg.onnx").is_file():
|
||||
model = YOLO("yolov8n-seg.pt")
|
||||
model.export(format="onnx", imgsz=[480,640])
|
||||
onnx_model = onnx.load(open("yolov8n-seg.onnx", "rb"))
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from extra.utils import get_child, fetch, download_file
|
|||
from pathlib import Path
|
||||
import cv2
|
||||
from collections import defaultdict
|
||||
import os
|
||||
import time, io, sys
|
||||
from tinygrad.nn.state import safe_load, load_state_dict
|
||||
|
||||
|
|
@ -398,13 +397,12 @@ if __name__ == '__main__':
|
|||
yolo_variant = sys.argv[2] if len(sys.argv) >= 3 else (print("No variant given, so choosing 'n' as the default. Yolov8 has different variants, you can choose from ['n', 's', 'm', 'l', 'x']") or 'n')
|
||||
print(f'running inference for YOLO version {yolo_variant}')
|
||||
|
||||
output_folder_path = './outputs_yolov8'
|
||||
if not os.path.exists(output_folder_path):
|
||||
os.makedirs(output_folder_path)
|
||||
output_folder_path = Path('./outputs_yolov8')
|
||||
output_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
#absolute image path or URL
|
||||
image_location = [np.frombuffer(io.BytesIO(fetch(img_path)).read(), np.uint8)]
|
||||
image = [cv2.imdecode(image_location[0], 1)]
|
||||
out_paths = [os.path.join(output_folder_path, img_path.split("/")[-1].split('.')[0] + "_output" + '.' + img_path.split("/")[-1].split('.')[1])]
|
||||
out_paths = [(output_folder_path / f"{Path(img_path).stem}_output{Path(img_path).suffix}").as_posix()]
|
||||
if not isinstance(image[0], np.ndarray):
|
||||
print('Error in image loading. Check your image file.')
|
||||
sys.exit(1)
|
||||
|
|
@ -414,7 +412,7 @@ if __name__ == '__main__':
|
|||
depth, width, ratio = get_variant_multiples(yolo_variant)
|
||||
yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)
|
||||
|
||||
weights_location = Path(__file__).parent.parent / "weights" / f'yolov8{yolo_variant}.safetensors'
|
||||
weights_location = Path(__file__).parents[1] / "weights" / f'yolov8{yolo_variant}.safetensors'
|
||||
download_file(f'https://gitlab.com/r3sist/yolov8_weights/-/raw/master/yolov8{yolo_variant}.safetensors', weights_location)
|
||||
|
||||
state_dict = safe_load(weights_location)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
from pathlib import Path
|
||||
from ctypes import *
|
||||
import json
|
||||
import collections
|
||||
|
|
@ -8,13 +8,13 @@ import faulthandler
|
|||
import struct
|
||||
faulthandler.enable()
|
||||
|
||||
basedir = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
|
||||
basedir = Path(__file__).resolve().parent
|
||||
|
||||
libane = None
|
||||
aneregs = None
|
||||
def init_libane():
|
||||
global libane, aneregs
|
||||
libane = cdll.LoadLibrary(os.path.join(basedir, "libane.dylib"))
|
||||
libane = cdll.LoadLibrary((basedir / "libane.dylib").as_posix())
|
||||
|
||||
libane.ANE_Compile.argtypes = [c_char_p, c_int]
|
||||
libane.ANE_Compile.restype = c_void_p
|
||||
|
|
@ -29,7 +29,7 @@ def init_libane():
|
|||
|
||||
#libane.ANE_RegDebug.restype = c_char_p
|
||||
|
||||
with open(os.path.join(basedir, "aneregs.json")) as f:
|
||||
with open(basedir / "aneregs.json") as f:
|
||||
aneregs = json.load(f)
|
||||
|
||||
ANE_Struct = [
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import numpy as np
|
||||
from PIL import Image
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.append(os.getcwd())
|
||||
sys.path.append(os.path.join(os.getcwd(), 'test'))
|
||||
cwd = Path.cwd()
|
||||
sys.path.append(cwd.as_posix())
|
||||
sys.path.append((cwd / 'test').as_posix())
|
||||
from extra.datasets import fetch_mnist
|
||||
from tqdm import trange
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
import os, random, gzip, tarfile, pickle
|
||||
import random, gzip, tarfile, pickle
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from tinygrad.tensor import Tensor
|
||||
from tinygrad.helpers import dtypes
|
||||
from extra.utils import download_file
|
||||
|
||||
def fetch_mnist():
|
||||
parse = lambda file: np.frombuffer(gzip.open(file).read(), dtype=np.uint8).copy()
|
||||
X_train = parse(os.path.dirname(__file__)+"/mnist/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
|
||||
Y_train = parse(os.path.dirname(__file__)+"/mnist/train-labels-idx1-ubyte.gz")[8:]
|
||||
X_test = parse(os.path.dirname(__file__)+"/mnist/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
|
||||
Y_test = parse(os.path.dirname(__file__)+"/mnist/t10k-labels-idx1-ubyte.gz")[8:]
|
||||
dirname = Path(__file__).parent.resolve()
|
||||
X_train = parse(dirname / "mnist/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
|
||||
Y_train = parse(dirname / "mnist/train-labels-idx1-ubyte.gz")[8:]
|
||||
X_test = parse(dirname / "mnist/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
|
||||
Y_test = parse(dirname / "mnist/t10k-labels-idx1-ubyte.gz")[8:]
|
||||
return X_train, Y_train, X_test, Y_test
|
||||
|
||||
cifar_mean = [0.4913997551666284, 0.48215855929893703, 0.4465309133731618]
|
||||
|
|
@ -31,7 +33,7 @@ def fetch_cifar(shuffle=False):
|
|||
Y[idx:idx+bs].assign(y[order])
|
||||
idx += bs
|
||||
return X, Y
|
||||
fn = os.path.dirname(__file__)+"/cifar-10-python.tar.gz"
|
||||
fn = Path(__file__).parent.resolve() / "cifar-10-python.tar.gz"
|
||||
download_file('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz', fn)
|
||||
tt = tarfile.open(fn, mode='r:gz')
|
||||
db = [pickle.load(tt.extractfile(f'cifar-10-batches-py/data_batch_{i}'), encoding="bytes") for i in range(1,6)]
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ def iterate(tokenizer, start=0):
|
|||
yield features, example
|
||||
|
||||
if __name__ == "__main__":
|
||||
tokenizer = BertTokenizer(str(Path(__file__).parent.parent.parent / "weights" / "bert_vocab.txt"))
|
||||
tokenizer = BertTokenizer(str(Path(__file__).parents[2] / "weights" / "bert_vocab.txt"))
|
||||
|
||||
X, Y = next(iterate(tokenizer))
|
||||
print(" ".join(X[0]["tokens"]))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import pickle
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
import tempfile, platform, os
|
||||
import tempfile, platform
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from tinygrad.helpers import prod, getenv, DEBUG, dtypes
|
||||
from tinygrad.ops import GlobalCounters
|
||||
|
|
@ -11,7 +12,7 @@ from tinygrad.shape.shapetracker import strides_for_shape
|
|||
OSX = platform.system() == "Darwin"
|
||||
WINDOWS = platform.system() == "Windows"
|
||||
|
||||
def temp(x:str) -> str: return os.path.join(tempfile.gettempdir(), x)
|
||||
def temp(x:str) -> str: return (Path(tempfile.gettempdir()) / x).as_posix()
|
||||
|
||||
def fetch(url):
|
||||
if url.startswith("/") or url.startswith("."):
|
||||
|
|
@ -33,19 +34,18 @@ def fetch_as_file(url):
|
|||
return fp
|
||||
|
||||
def download_file(url, fp, skip_if_exists=True):
|
||||
import requests, pathlib
|
||||
if skip_if_exists and os.path.isfile(fp) and os.stat(fp).st_size > 0:
|
||||
import requests
|
||||
if skip_if_exists and Path(fp).is_file() and Path(fp).stat().st_size > 0:
|
||||
return
|
||||
r = requests.get(url, stream=True)
|
||||
assert r.status_code == 200
|
||||
progress_bar = tqdm(total=int(r.headers.get('content-length', 0)), unit='B', unit_scale=True, desc=url)
|
||||
(path := pathlib.Path(fp).parent).mkdir(parents=True, exist_ok=True)
|
||||
(path := Path(fp).parent).mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=path, delete=False) as f:
|
||||
for chunk in r.iter_content(chunk_size=16384):
|
||||
progress_bar.update(f.write(chunk))
|
||||
f.close()
|
||||
os.rename(f.name, fp)
|
||||
|
||||
Path(f.name).rename(fp)
|
||||
|
||||
def my_unpickle(fb0):
|
||||
key_prelookup = defaultdict(list)
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ class BertForQuestionAnswering:
|
|||
self.qa_outputs = Linear(hidden_size, 2)
|
||||
|
||||
def load_from_pretrained(self):
|
||||
fn = Path(__file__).parent.parent / "weights/bert_for_qa.pt"
|
||||
fn = Path(__file__).parents[1] / "weights/bert_for_qa.pt"
|
||||
download_file("https://zenodo.org/record/3733896/files/model.pytorch?download=1", fn)
|
||||
fn_vocab = Path(__file__).parent.parent / "weights/bert_vocab.txt"
|
||||
fn_vocab = Path(__file__).parents[1] / "weights/bert_vocab.txt"
|
||||
download_file("https://zenodo.org/record/3733896/files/vocab.txt?download=1", fn_vocab)
|
||||
|
||||
import torch
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ class RNNT:
|
|||
return out.realize()
|
||||
|
||||
def load_from_pretrained(self):
|
||||
fn = Path(__file__).parent.parent / "weights/rnnt.pt"
|
||||
fn = Path(__file__).parents[1] / "weights/rnnt.pt"
|
||||
download_file("https://zenodo.org/record/3662521/files/DistributedDataParallel_1576581068.9962234-epoch-100.pt?download=1", fn)
|
||||
|
||||
import torch
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class UNet3D:
|
|||
return x
|
||||
|
||||
def load_from_pretrained(self):
|
||||
fn = Path(__file__).parent.parent / "weights" / "unet-3d.ckpt"
|
||||
fn = Path(__file__).parents[1] / "weights" / "unet-3d.ckpt"
|
||||
download_file("https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch.ptc?download=1", fn)
|
||||
state_dict = torch.jit.load(fn, map_location=torch.device("cpu")).state_dict()
|
||||
for k, v in state_dict.items():
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
import os, time, io, pathlib, sys, traceback
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parents[1]))
|
||||
|
||||
if os.getenv("OPT", None) is None:
|
||||
os.environ['OPT'] = '99'
|
||||
|
|
|
|||
6
setup.py
6
setup.py
|
|
@ -1,10 +1,10 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from setuptools import setup
|
||||
|
||||
directory = os.path.abspath(os.path.dirname(__file__))
|
||||
with open(os.path.join(directory, 'README.md'), encoding='utf-8') as f:
|
||||
directory = Path(__file__).resolve().parent
|
||||
with open(directory / 'README.md', encoding='utf-8') as f:
|
||||
long_description = f.read()
|
||||
|
||||
setup(name='tinygrad',
|
||||
|
|
|
|||
5
sz.py
5
sz.py
|
|
@ -1,5 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
from pathlib import Path
|
||||
import token
|
||||
import tokenize
|
||||
import itertools
|
||||
|
|
@ -13,11 +14,11 @@ if __name__ == "__main__":
|
|||
for path, subdirs, files in os.walk("tinygrad"):
|
||||
for name in files:
|
||||
if not name.endswith(".py"): continue
|
||||
filepath = os.path.join(path, name)
|
||||
filepath = Path(path) / name
|
||||
with tokenize.open(filepath) as file_:
|
||||
tokens = [t for t in tokenize.generate_tokens(file_.readline) if t.type in TOKEN_WHITELIST]
|
||||
token_count, line_count = len(tokens), len(set([t.start[0] for t in tokens]))
|
||||
table.append([filepath, line_count, token_count/line_count])
|
||||
table.append([filepath.as_posix(), line_count, token_count/line_count])
|
||||
|
||||
print(tabulate([headers] + sorted(table, key=lambda x: -x[1]), headers="firstrow", floatfmt=".1f")+"\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ def benchmark(mnm, nm, fxn):
|
|||
CSV[nm] = min(tms)*1e-6
|
||||
return min(tms), ret
|
||||
|
||||
#BASE = pathlib.Path(__file__).parent.parent.parent / "weights" / "onnx"
|
||||
#BASE = pathlib.Path(__file__).parents[2] / "weights" / "onnx"
|
||||
BASE = pathlib.Path("/tmp/onnx")
|
||||
def benchmark_model(m, validate_outs=False):
|
||||
global open_csv, CSV
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from tinygrad.nn.state import safe_load, load_state_dict
|
|||
class TestYOLOv8(unittest.TestCase):
|
||||
def test_all_load_weights(self):
|
||||
for variant in ['n', 's', 'm', 'l', 'x']:
|
||||
weights_location = Path(__file__).parent.parent.parent / "weights" / f'yolov8{variant}.safetensors'
|
||||
weights_location = Path(__file__).parents[2] / "weights" / f'yolov8{variant}.safetensors'
|
||||
download_file(f'https://gitlab.com/r3sist/yolov8_weights/-/raw/master/yolov8{variant}.safetensors', weights_location)
|
||||
|
||||
depth, width, ratio = get_variant_multiples(variant)
|
||||
|
|
@ -23,7 +23,7 @@ class TestYOLOv8(unittest.TestCase):
|
|||
def test_predictions(self):
|
||||
test_image_urls = ['https://raw.githubusercontent.com/ultralytics/yolov5/master/data/images/bus.jpg', 'https://www.aljazeera.com/wp-content/uploads/2022/10/2022-04-28T192650Z_1186456067_UP1EI4S1I0P14_RTRMADP_3_SOCCER-ENGLAND-MUN-CHE-REPORT.jpg']
|
||||
variant = 'n'
|
||||
weights_location = Path(__file__).parent.parent.parent / "weights" / f'yolov8{variant}.safetensors'
|
||||
weights_location = Path(__file__).parents[2] / "weights" / f'yolov8{variant}.safetensors'
|
||||
depth, width, ratio = get_variant_multiples(variant)
|
||||
TinyYolov8 = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)
|
||||
state_dict = safe_load(weights_location)
|
||||
|
|
@ -40,13 +40,13 @@ class TestYOLOv8(unittest.TestCase):
|
|||
|
||||
def test_forward_pass_torch_onnx(self):
|
||||
variant = 'n'
|
||||
weights_location_onnx = Path(__file__).parent.parent.parent / "weights" / f'yolov8{variant}.onnx'
|
||||
weights_location_pt = Path(__file__).parent.parent.parent / "weights" / f'yolov8{variant}.pt'
|
||||
weights_location = Path(__file__).parent.parent.parent / "weights" / f'yolov8{variant}.safetensors'
|
||||
weights_location_onnx = Path(__file__).parents[2] / "weights" / f'yolov8{variant}.onnx'
|
||||
weights_location_pt = Path(__file__).parents[2] / "weights" / f'yolov8{variant}.pt'
|
||||
weights_location = Path(__file__).parents[2] / "weights" / f'yolov8{variant}.safetensors'
|
||||
|
||||
download_file(f'https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8{variant}.pt', weights_location_pt)
|
||||
# the ultralytics export prints a lot of unneccesary things
|
||||
if not os.path.isfile(weights_location_onnx):
|
||||
if not weights_location_onnx.is_file():
|
||||
model = ultralytics.YOLO(model=weights_location_pt, task='Detect')
|
||||
model.export(format="onnx",imgsz=[640, 480])
|
||||
|
||||
|
|
|
|||
|
|
@ -64,10 +64,10 @@ class TestDownloadFile(unittest.TestCase):
|
|||
mock_response.status_code = 200
|
||||
mock_response.headers = {'content-length': '8'}
|
||||
mock_requests.return_value = mock_response
|
||||
self.assertFalse(os.path.exists(self.test_file.parent))
|
||||
self.assertFalse(self.test_file.parent.exists())
|
||||
download_file("https://www.mock.com/fake.txt", self.test_file, skip_if_exists=False)
|
||||
self.assertTrue(os.path.exists(self.test_file.parent))
|
||||
self.assertTrue(os.path.isfile(self.test_file))
|
||||
self.assertTrue(self.test_file.parent.exists())
|
||||
self.assertTrue(self.test_file.is_file())
|
||||
self.assertEqual('12345678', self.test_file.read_text())
|
||||
|
||||
class TestUtils(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ class TestTorchLoad(unittest.TestCase):
|
|||
# TODO: support pytorch tar format with minimal lines
|
||||
#def test_load_resnet(self): compare_weights_both('https://download.pytorch.org/models/resnet50-19c8e357.pth')
|
||||
|
||||
test_fn = pathlib.Path(__file__).parent.parent.parent / "weights/LLaMA/7B/consolidated.00.pth"
|
||||
test_fn = pathlib.Path(__file__).parents[2] / "weights/LLaMA/7B/consolidated.00.pth"
|
||||
#test_size = test_fn.stat().st_size
|
||||
test_size = 1024*1024*1024*2
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import subprocess, time, re, hashlib, tempfile, os, functools
|
||||
import subprocess, time, re, hashlib, tempfile, functools
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import numpy as np
|
||||
from pycuda.compiler import compile as cuda_compile # type: ignore
|
||||
|
|
@ -66,7 +67,7 @@ class CUDAProgram:
|
|||
if DEBUG >= 5: print(pretty_ptx(prg))
|
||||
if DEBUG >= 6:
|
||||
try:
|
||||
fn = os.path.join(tempfile.gettempdir(), f"tinycuda_{hashlib.md5(prg.encode('utf-8')).hexdigest()}")
|
||||
fn = (Path(tempfile.gettempdir()) / f"tinycuda_{hashlib.md5(prg.encode('utf-8')).hexdigest()}").as_posix()
|
||||
with open(fn + ".ptx", "wb") as f: f.write(prg.encode('utf-8'))
|
||||
subprocess.run(["ptxas", f"-arch={arch()}", "-o", fn, fn+".ptx"], check=True)
|
||||
print(subprocess.check_output(['nvdisasm', fn]).decode('utf-8'))
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ OSX_TIMING_RATIO = (125/3) if OSX else 1.0 # see test/external_osx_profiling.p
|
|||
|
||||
# TODO: if you fork and exit the child process after creating anything with cl on AMD, it hangs on e.wait()
|
||||
ROCM_LLVM_PATH = pathlib.Path("/opt/rocm/llvm/bin")
|
||||
#ROCM_LLVM_PATH = pathlib.Path(__file__).parent.parent.parent.parent / "extra/rocm/build/llvm-project/bin"
|
||||
#ROCM_LLVM_PATH = pathlib.Path(__file__).parents[3] / "extra/rocm/build/llvm-project/bin"
|
||||
if DEBUG >= 5:
|
||||
early_exec = fromimport("extra.helpers", "enable_early_exec")()
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ class MetalProgram:
|
|||
unwrap(arc.addComputePipelineFunctionsWithDescriptor_error_(desc, None))
|
||||
unwrap(arc.serializeToURL_error_(Cocoa.NSURL.URLWithString_("file:///tmp/shader.bin"), None))
|
||||
# clone https://github.com/dougallj/applegpu.git in tinygrad/disassemblers
|
||||
os.system(f"cd {pathlib.Path(__file__).parent.parent.parent}/disassemblers/applegpu && python3 compiler_explorer.py /tmp/shader.bin")
|
||||
os.system(f"cd {pathlib.Path(__file__).parents[2]}/disassemblers/applegpu && python3 compiler_explorer.py /tmp/shader.bin")
|
||||
self.pipeline_state = unwrap(METAL.device.newComputePipelineStateWithFunction_error_(self.fxn, None))
|
||||
|
||||
def __call__(self, global_size, local_size, *bufs, wait=False):
|
||||
|
|
|
|||
Loading…
Reference in New Issue