working

movinet/data.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+import random
+from typing import Literal
+import cv2
+import numpy as np
+import tensorflow as tf
+
+TRAINING_RATIO = 0.1
+VALIDATION_RATIO = 0.01
+
+def format_frames(frame, output_size):
+    frame = tf.image.convert_image_dtype(frame, tf.float32)
+    frame = tf.image.resize_with_pad(frame, *output_size)
+    return frame
+
+def frames_from_video_file(video_path: str, n_frames: int, output_size=(256, 256), frame_step=15):
+    capture = cv2.VideoCapture(video_path)
+    if not capture.isOpened(): raise ValueError('Video file could not be opened.')
+    total_frames = capture.get(cv2.CAP_PROP_FRAME_COUNT)
+    need_frames = 1 + (n_frames - 1) * frame_step
+
+    if need_frames <= total_frames:
+        start = random.randint(0, total_frames - need_frames + 1)
+        capture.set(cv2.CAP_PROP_POS_FRAMES, start)
+
+    frames = []
+    for _ in range(n_frames - 1):
+        for _ in range(frame_step):
+            ok, frame = capture.read()
+        if ok:
+            frames.append(format_frames(frame, output_size))
+        else:
+            frames.append(np.zeros((output_size[0], output_size[1], 3)))
+    capture.release()
+
+    frames = np.array(frames)
+    frames = frames[..., [2, 1, 0]]
+    return frames
+
+def Data(data_dir: Path):
+    return {
+        'training':{
+            a.name: (lambda ps: ps[:int(len(ps) * TRAINING_RATIO)])([x for x in a.iterdir()])
+            for a in data_dir.iterdir()
+        },
+        'validation': {
+            a.name: (lambda ps: ps[
+                int(len(ps) * TRAINING_RATIO) :
+                int(len(ps) * (TRAINING_RATIO + VALIDATION_RATIO))
+            ])([x for x in a.iterdir()])
+            for a in data_dir.iterdir()
+        },
+    }
+
+def frame_generator(data_dir: Path, n_frames: int, split: Literal['training', 'validation']):
+    class_names = sorted([x.name for x in data_dir.iterdir()])
+    class_ids_for_name = {
+        name: i
+        for i, name in enumerate(class_names)
+    }
+    data = Data(data_dir)
+
+    def generator():
+        pairs = [
+            (path, name)
+            for name, paths in data[split].items()
+            for path in paths
+        ]
+        random.shuffle(pairs)
+        for path, name in pairs:
+            video_frames = frames_from_video_file(str(path), n_frames)
+            label = class_ids_for_name[name]
+            yield video_frames, label
+    return generator
+
+def total_steps(data_dir: Path):
+    data = Data(data_dir)
+    size = lambda d: sum([len(x) for x in d.values()])
+    return size(data['training']), size(data['validation'])

movinet/model.py

@@ -0,0 +1,9 @@
+from site_packages.models.official.projects.movinet.modeling import movinet_model
+
+def build_classifier(batch_size: int, num_frames: int, resolution: int, backbone, num_classes: int):
+    model = movinet_model.MovinetClassifier(
+        backbone=backbone,
+        num_classes=num_classes,
+    )
+    model.build([batch_size, num_frames, resolution, resolution, 3])
+    return model

movinet/scripts/train.py

@@ -0,0 +1,58 @@
+import os
+from pathlib import Path
+
+import tensorflow as tf
+import tf_keras as keras
+
+from site_packages.models.official.projects.movinet.modeling import movinet
+
+from movinet.data import frame_generator, total_steps
+from movinet.model import build_classifier
+
+model_id = 'a0'
+resolution = 256
+batch_size = 8
+num_frames = 8
+num_classes = 6
+model_save_path = "out/aero-recognize-classifier.keras"
+num_epochs = 2
+
+print('Load data.')
+data_dir = Path('assets/datasets/Aero')
+output_signature = (
+    tf.TensorSpec(shape=(None, None, None, 3), dtype=tf.float32),
+    tf.TensorSpec(shape=(), dtype=tf.int16),
+)
+training_data = tf.data.Dataset.from_generator(
+    frame_generator(data_dir, num_frames, 'training'),
+    output_signature=output_signature,
+)
+training_data = training_data.batch(batch_size)
+validation_data = tf.data.Dataset.from_generator(
+    frame_generator(data_dir, num_frames, 'validation'),
+    output_signature=output_signature,
+)
+validation_data = validation_data.batch(batch_size)
+
+print('Build model.')
+backbone = movinet.Movinet(model_id=model_id)
+backbone.trainable = True
+model = build_classifier(batch_size, num_frames, resolution, backbone, 6)
+
+print('Start training.')
+model_dir = os.path.dirname(model_save_path)
+save_model = keras.callbacks.ModelCheckpoint(filepath=model_save_path)
+loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+# optimizer = keras.optimizers.legacy.Adam(learning_rate=0.001)
+model.compile(optimizer='adam', loss=loss, metrics=['accuracy'])
+train_steps, validation_steps = total_steps(data_dir)
+results = model.fit(
+    training_data,
+    steps_per_epoch=train_steps,
+    validation_data=validation_data,
+    validation_steps=validation_steps,
+    epochs=num_epochs,
+    validation_freq=1,
+    verbose=1,
+    callbacks=[save_model],
+)

playgrounds/main.py

@@ -1,3 +0,0 @@
-from playgrounds.yolo import main
-
-main()

playgrounds/verify_metal.py

@@ -0,0 +1,14 @@
+import tensorflow as tf
+
+cifar = tf.keras.datasets.cifar100
+(x_train, y_train), (x_test, y_test) = cifar.load_data()
+model = tf.keras.applications.ResNet50(
+    include_top=True,
+    weights=None,
+    input_shape=(32, 32, 3),
+    classes=100,
+)
+
+loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)
+model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+model.fit(x_train, y_train, epochs=5, batch_size=64)

requirements.txt

@@ -0,0 +1,35 @@
+gradio
+tensorflow
+opencv-python
+
+# cspell: disable
+# models/official
+six
+google-api-python-client>=1.6.7
+kaggle>=1.3.9
+numpy>=1.20
+oauth2client
+pandas>=0.22.0
+psutil>=5.4.3
+py-cpuinfo>=3.3.0
+scipy>=0.19.1
+tensorflow-hub>=0.6.0
+tensorflow-model-optimization>=0.4.1
+tensorflow-datasets
+tf-keras
+gin-config
+tf_slim>=1.1.0
+Cython
+matplotlib
+# Loader becomes a required positional argument in 6.0 in yaml.load
+pyyaml>=6.0.0
+# CV related dependencies
+opencv-python-headless
+Pillow
+pycocotools
+# NLP related dependencies
+seqeval
+sentencepiece
+sacrebleu
+# Projects/vit dependencies
+immutabledict

site_packages/models

@@ -0,0 +1 @@
+Subproject commit d14cf43b09cc29d68900bb9f766de19b01acde40