sanchit-gandhi HF staff commited on
Commit
b0782ac
1 Parent(s): ea31514

dockerfile

Browse files
Files changed (8) hide show
  1. Dockerfile +30 -0
  2. README.md +6 -8
  3. app.py +0 -235
  4. nginx.conf +23 -0
  5. packages.txt +0 -2
  6. processing_whisper.py +0 -146
  7. requirements.txt +0 -4
  8. run.sh +6 -0
Dockerfile ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM ubuntu
2
+
3
+ # Based on https://huggingface.co/spaces/radames/nginx-gradio-reverse-proxy/blob/main/Dockerfile
4
+ USER root
5
+
6
+ RUN apt-get -y update && apt-get -y install nginx
7
+ RUN mkdir -p /var/cache/nginx \
8
+ /var/log/nginx \
9
+ /var/lib/nginx
10
+ RUN touch /var/run/nginx.pid
11
+
12
+ RUN chown -R 1000:1000 /var/cache/nginx \
13
+ /var/log/nginx \
14
+ /var/lib/nginx \
15
+ /var/run/nginx.pid
16
+
17
+ RUN useradd -m -u 1000 user
18
+
19
+ USER user
20
+ ENV HOME=/home/user
21
+
22
+ RUN mkdir $HOME/app
23
+ WORKDIR $HOME/app
24
+
25
+ # Copy nginx configuration
26
+ COPY --chown=user nginx.conf /etc/nginx/sites-available/default
27
+ COPY --chown=user . .
28
+
29
+ CMD ["bash", "run.sh"]
30
+
README.md CHANGED
@@ -1,12 +1,10 @@
1
  ---
2
- title: Whisper JAX
3
- emoji: ⚡️
4
- colorFrom: yellow
5
- colorTo: indigo
6
- sdk: gradio
7
- sdk_version: 3.27.0
8
- app_file: app.py
9
  pinned: false
10
  ---
11
 
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Whisper PoC
3
+ emoji: 📉
4
+ colorFrom: gray
5
+ colorTo: pink
6
+ sdk: docker
 
 
7
  pinned: false
8
  ---
9
 
10
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py DELETED
@@ -1,235 +0,0 @@
1
- import base64
2
- import math
3
- import os
4
- import time
5
- from functools import partial
6
- from multiprocessing import Pool
7
-
8
- import gradio as gr
9
- import numpy as np
10
- import pytube
11
- import requests
12
- from processing_whisper import WhisperPrePostProcessor
13
- from transformers.models.whisper.tokenization_whisper import TO_LANGUAGE_CODE
14
- from transformers.pipelines.audio_utils import ffmpeg_read
15
-
16
-
17
- title = "Whisper JAX: The Fastest Whisper API ⚡️"
18
-
19
- description = """Whisper JAX is an optimised implementation of the [Whisper model](https://huggingface.co/openai/whisper-large-v2) by OpenAI. It runs on JAX with a TPU v4-8 in the backend. Compared to PyTorch on an A100 GPU, it is over [**70x faster**](https://github.com/sanchit-gandhi/whisper-jax#benchmarks), making it the fastest Whisper API available.
20
-
21
- Note that at peak times, you may find yourself in the queue for this demo. When you submit a request, your queue position will be shown in the top right-hand side of the demo pane. Once you reach the front of the queue, your audio file will be sent to the TPU and then transcribed, with the progress displayed through a progress bar.
22
-
23
- To skip the queue, you may wish to create your own inference endpoint, details for which can be found in the [Whisper JAX repository](https://github.com/sanchit-gandhi/whisper-jax#creating-an-endpoint).
24
- """
25
-
26
- article = "Whisper large-v2 model by OpenAI. Backend running JAX on a TPU v4-8 through the generous support of the [TRC](https://sites.research.google/trc/about/) programme. Whisper JAX [code](https://github.com/sanchit-gandhi/whisper-jax) and Gradio demo by 🤗 Hugging Face."
27
-
28
- API_SEND_URL = os.getenv("API_SEND_URL")
29
- API_FORWARD_URL = os.getenv("API_FORWARD_URL")
30
-
31
- language_names = sorted(TO_LANGUAGE_CODE.keys())
32
- CHUNK_LENGTH_S = 30
33
- BATCH_SIZE = 16
34
- NUM_PROC = 16
35
- FILE_LIMIT_MB = 1000
36
-
37
-
38
- def query(url, payload):
39
- response = requests.post(url, json=payload)
40
- return response.json(), response.status_code
41
-
42
-
43
- def inference(batch_id, idx, task=None, return_timestamps=False):
44
- payload = {"batch_id": batch_id, "idx": idx, "task": task, "return_timestamps": return_timestamps}
45
-
46
- data, status_code = query(API_FORWARD_URL, payload)
47
-
48
- if status_code == 200:
49
- tokens = {"tokens": np.asarray(data["tokens"])}
50
- return tokens
51
- else:
52
- gr.Error(data["detail"])
53
-
54
-
55
- def send_chunks(batch, batch_id):
56
- feature_shape = batch["input_features"].shape
57
- batch["input_features"] = base64.b64encode(batch["input_features"].tobytes()).decode()
58
- query(API_SEND_URL, {"batch": batch, "feature_shape": feature_shape, "batch_id": batch_id})
59
-
60
-
61
- def forward(batch_id, idx, task=None, return_timestamps=False):
62
- outputs = inference(batch_id, idx, task, return_timestamps)
63
- return outputs
64
-
65
-
66
- # Copied from https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/utils.py#L50
67
- def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."):
68
- if seconds is not None:
69
- milliseconds = round(seconds * 1000.0)
70
-
71
- hours = milliseconds // 3_600_000
72
- milliseconds -= hours * 3_600_000
73
-
74
- minutes = milliseconds // 60_000
75
- milliseconds -= minutes * 60_000
76
-
77
- seconds = milliseconds // 1_000
78
- milliseconds -= seconds * 1_000
79
-
80
- hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""
81
- return f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}"
82
- else:
83
- # we have a malformed timestamp so just return it as is
84
- return seconds
85
-
86
-
87
- if __name__ == "__main__":
88
- processor = WhisperPrePostProcessor.from_pretrained("openai/whisper-large-v2")
89
- stride_length_s = CHUNK_LENGTH_S / 6
90
- chunk_len = round(CHUNK_LENGTH_S * processor.feature_extractor.sampling_rate)
91
- stride_left = stride_right = round(stride_length_s * processor.feature_extractor.sampling_rate)
92
- step = chunk_len - stride_left - stride_right
93
- pool = Pool(NUM_PROC)
94
-
95
- def tqdm_generate(inputs: dict, task: str, return_timestamps: bool, progress: gr.Progress):
96
- inputs_len = inputs["array"].shape[0]
97
- all_chunk_start_batch_id = np.arange(0, inputs_len, step)
98
- num_samples = len(all_chunk_start_batch_id)
99
- num_batches = math.ceil(num_samples / BATCH_SIZE)
100
- dummy_batches = list(range(num_batches))
101
-
102
- dataloader = processor.preprocess_batch(inputs, chunk_length_s=CHUNK_LENGTH_S, batch_size=BATCH_SIZE)
103
- progress(0, desc="Sending audio to TPU...")
104
- batch_id = np.random.randint(
105
- 1000000
106
- ) # TODO(SG): swap to an iterator - currently taking our 1 in a million chances
107
- pool.map(partial(send_chunks, batch_id=batch_id), dataloader)
108
-
109
- model_outputs = []
110
- start_time = time.time()
111
- # iterate over our chunked audio samples
112
- for idx in progress.tqdm(dummy_batches, desc="Transcribing..."):
113
- model_outputs.append(forward(batch_id, idx, task=task, return_timestamps=return_timestamps))
114
- runtime = time.time() - start_time
115
-
116
- post_processed = processor.postprocess(model_outputs, return_timestamps=return_timestamps)
117
- text = post_processed["text"]
118
- timestamps = post_processed.get("chunks")
119
- if timestamps is not None:
120
- timestamps = [
121
- f"[{format_timestamp(chunk['timestamp'][0])} -> {format_timestamp(chunk['timestamp'][1])}] {chunk['text']}"
122
- for chunk in timestamps
123
- ]
124
- text = "\n".join(str(feature) for feature in timestamps)
125
- return text, runtime
126
-
127
- def transcribe_chunked_audio(inputs, task, return_timestamps, progress=gr.Progress()):
128
- progress(0, desc="Loading audio file...")
129
- if inputs is None:
130
- raise gr.Error("No audio file submitted! Please upload an audio file before submitting your request.")
131
- file_size_mb = os.stat(inputs).st_size / (1024 * 1024)
132
- if file_size_mb > FILE_LIMIT_MB:
133
- raise gr.Error(
134
- f"File size exceeds file size limit. Got file of size {file_size_mb:.2f}MB for a limit of {FILE_LIMIT_MB}MB."
135
- )
136
-
137
- with open(inputs, "rb") as f:
138
- inputs = f.read()
139
-
140
- inputs = ffmpeg_read(inputs, processor.feature_extractor.sampling_rate)
141
- inputs = {"array": inputs, "sampling_rate": processor.feature_extractor.sampling_rate}
142
- text, runtime = tqdm_generate(inputs, task=task, return_timestamps=return_timestamps, progress=progress)
143
- return text, runtime
144
-
145
- def _return_yt_html_embed(yt_url):
146
- video_id = yt_url.split("?v=")[-1]
147
- HTML_str = (
148
- f'<center> <iframe width="500" height="320" src="https://www.youtube.com/embed/{video_id}"> </iframe>'
149
- " </center>"
150
- )
151
- return HTML_str
152
-
153
- def transcribe_youtube(yt_url, task, return_timestamps, progress=gr.Progress(), max_filesize=75.0):
154
- progress(0, desc="Loading audio file...")
155
- html_embed_str = _return_yt_html_embed(yt_url)
156
- try:
157
- yt = pytube.YouTube(yt_url)
158
- stream = yt.streams.filter(only_audio=True)[0]
159
- except:
160
- raise gr.Error("An error occurred while loading the YouTube video. Please try again.")
161
-
162
- if stream.filesize_mb > max_filesize:
163
- raise gr.Error(f"Maximum YouTube file size is {max_filesize}MB, got {stream.filesize_mb:.2f}MB.")
164
-
165
- stream.download(filename="audio.mp3")
166
-
167
- with open("audio.mp3", "rb") as f:
168
- inputs = f.read()
169
-
170
- inputs = ffmpeg_read(inputs, processor.feature_extractor.sampling_rate)
171
- inputs = {"array": inputs, "sampling_rate": processor.feature_extractor.sampling_rate}
172
- text, runtime = tqdm_generate(inputs, task=task, return_timestamps=return_timestamps, progress=progress)
173
- return html_embed_str, text, runtime
174
-
175
- microphone_chunked = gr.Interface(
176
- fn=transcribe_chunked_audio,
177
- inputs=[
178
- gr.inputs.Audio(source="microphone", optional=True, type="filepath"),
179
- gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
180
- gr.inputs.Checkbox(default=False, label="Return timestamps"),
181
- ],
182
- outputs=[
183
- gr.outputs.Textbox(label="Transcription").style(show_copy_button=True),
184
- gr.outputs.Textbox(label="Transcription Time (s)"),
185
- ],
186
- allow_flagging="never",
187
- title=title,
188
- description=description,
189
- article=article,
190
- )
191
-
192
- audio_chunked = gr.Interface(
193
- fn=transcribe_chunked_audio,
194
- inputs=[
195
- gr.inputs.Audio(source="upload", optional=True, label="Audio file", type="filepath"),
196
- gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
197
- gr.inputs.Checkbox(default=False, label="Return timestamps"),
198
- ],
199
- outputs=[
200
- gr.outputs.Textbox(label="Transcription").style(show_copy_button=True),
201
- gr.outputs.Textbox(label="Transcription Time (s)"),
202
- ],
203
- allow_flagging="never",
204
- title=title,
205
- description=description,
206
- article=article,
207
- )
208
-
209
- youtube = gr.Interface(
210
- fn=transcribe_youtube,
211
- inputs=[
212
- gr.inputs.Textbox(lines=1, placeholder="Paste the URL to a YouTube video here", label="YouTube URL"),
213
- gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
214
- gr.inputs.Checkbox(default=False, label="Return timestamps"),
215
- ],
216
- outputs=[
217
- gr.outputs.HTML(label="Video"),
218
- gr.outputs.Textbox(label="Transcription").style(show_copy_button=True),
219
- gr.outputs.Textbox(label="Transcription Time (s)"),
220
- ],
221
- allow_flagging="never",
222
- title=title,
223
- examples=[["https://www.youtube.com/watch?v=m8u-18Q0s7I", "transcribe", False]],
224
- cache_examples=False,
225
- description=description,
226
- article=article,
227
- )
228
-
229
- demo = gr.Blocks()
230
-
231
- with demo:
232
- gr.TabbedInterface([microphone_chunked, audio_chunked, youtube], ["Microphone", "Audio File", "YouTube"])
233
-
234
- demo.queue(max_size=10)
235
- demo.launch(show_api=False, max_threads=10)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
nginx.conf ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ server {
2
+ listen 7860 default_server;
3
+ listen [::]:7860 default_server;
4
+
5
+ root /usr/share/nginx/html;
6
+ index index.html index.htm;
7
+
8
+ server_name _;
9
+ location / {
10
+ proxy_pass http://API_URL;
11
+ proxy_set_header Host API_URL;
12
+ proxy_set_header X-Real-IP $remote_addr;
13
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
14
+ #proxy_set_header X-Forwarded-Proto $scheme;
15
+ proxy_set_header X-Forwarded-Proto http;
16
+ proxy_set_header X-Forwarded-Ssl off;
17
+ proxy_set_header X-Url-Scheme http;
18
+ proxy_buffering off;
19
+ proxy_http_version 1.1;
20
+ proxy_set_header Upgrade $http_upgrade;
21
+ proxy_set_header Connection "upgrade";
22
+ }
23
+ }
packages.txt DELETED
@@ -1,2 +0,0 @@
1
- ffmpeg
2
-
 
 
 
processing_whisper.py DELETED
@@ -1,146 +0,0 @@
1
- import math
2
-
3
- import numpy as np
4
- from transformers import WhisperProcessor
5
-
6
-
7
- class WhisperPrePostProcessor(WhisperProcessor):
8
- def chunk_iter_with_batch(self, inputs, chunk_len, stride_left, stride_right, batch_size):
9
- inputs_len = inputs.shape[0]
10
- step = chunk_len - stride_left - stride_right
11
-
12
- all_chunk_start_idx = np.arange(0, inputs_len, step)
13
- num_samples = len(all_chunk_start_idx)
14
-
15
- num_batches = math.ceil(num_samples / batch_size)
16
- batch_idx = np.array_split(np.arange(num_samples), num_batches)
17
-
18
- for i, idx in enumerate(batch_idx):
19
- chunk_start_idx = all_chunk_start_idx[idx]
20
-
21
- chunk_end_idx = chunk_start_idx + chunk_len
22
-
23
- chunks = [inputs[chunk_start:chunk_end] for chunk_start, chunk_end in zip(chunk_start_idx, chunk_end_idx)]
24
- processed = self.feature_extractor(
25
- chunks, sampling_rate=self.feature_extractor.sampling_rate, return_tensors="np"
26
- )
27
-
28
- _stride_left = np.where(chunk_start_idx == 0, 0, stride_left)
29
- is_last = np.where(stride_right > 0, chunk_end_idx > inputs_len, chunk_end_idx >= inputs_len)
30
- _stride_right = np.where(is_last, 0, stride_right)
31
-
32
- chunk_lens = [chunk.shape[0] for chunk in chunks]
33
- strides = [
34
- (int(chunk_l), int(_stride_l), int(_stride_r))
35
- for chunk_l, _stride_l, _stride_r in zip(chunk_lens, _stride_left, _stride_right)
36
- ]
37
-
38
- yield {"stride": strides, **processed}
39
-
40
- def preprocess_batch(self, inputs, chunk_length_s=0, stride_length_s=None, batch_size=None):
41
- stride = None
42
- if isinstance(inputs, dict):
43
- stride = inputs.pop("stride", None)
44
- # Accepting `"array"` which is the key defined in `datasets` for
45
- # better integration
46
- if not ("sampling_rate" in inputs and ("raw" in inputs or "array" in inputs)):
47
- raise ValueError(
48
- "When passing a dictionary to FlaxWhisperPipline, the dict needs to contain a "
49
- '"raw" or "array" key containing the numpy array representing the audio, and a "sampling_rate" key '
50
- "containing the sampling rate associated with the audio array."
51
- )
52
-
53
- _inputs = inputs.pop("raw", None)
54
- if _inputs is None:
55
- # Remove path which will not be used from `datasets`.
56
- inputs.pop("path", None)
57
- _inputs = inputs.pop("array", None)
58
- in_sampling_rate = inputs.pop("sampling_rate")
59
- inputs = _inputs
60
-
61
- if in_sampling_rate != self.feature_extractor.sampling_rate:
62
- try:
63
- import librosa
64
- except ImportError as err:
65
- raise ImportError(
66
- "To support resampling audio files, please install 'librosa' and 'soundfile'."
67
- ) from err
68
-
69
- inputs = librosa.resample(
70
- inputs, orig_sr=in_sampling_rate, target_sr=self.feature_extractor.sampling_rate
71
- )
72
- ratio = self.feature_extractor.sampling_rate / in_sampling_rate
73
- else:
74
- ratio = 1
75
-
76
- if not isinstance(inputs, np.ndarray):
77
- raise ValueError(f"We expect a numpy ndarray as input, got `{type(inputs)}`.")
78
- if len(inputs.shape) != 1:
79
- raise ValueError(
80
- f"We expect a single channel audio input for the Flax Whisper API, got {len(inputs.shape)} channels."
81
- )
82
-
83
- if stride is not None:
84
- if stride[0] + stride[1] > inputs.shape[0]:
85
- raise ValueError("Stride is too large for input.")
86
-
87
- # Stride needs to get the chunk length here, it's going to get
88
- # swallowed by the `feature_extractor` later, and then batching
89
- # can add extra data in the inputs, so we need to keep track
90
- # of the original length in the stride so we can cut properly.
91
- stride = (inputs.shape[0], int(round(stride[0] * ratio)), int(round(stride[1] * ratio)))
92
-
93
- if chunk_length_s:
94
- if stride_length_s is None:
95
- stride_length_s = chunk_length_s / 6
96
-
97
- if isinstance(stride_length_s, (int, float)):
98
- stride_length_s = [stride_length_s, stride_length_s]
99
-
100
- chunk_len = round(chunk_length_s * self.feature_extractor.sampling_rate)
101
- stride_left = round(stride_length_s[0] * self.feature_extractor.sampling_rate)
102
- stride_right = round(stride_length_s[1] * self.feature_extractor.sampling_rate)
103
-
104
- if chunk_len < stride_left + stride_right:
105
- raise ValueError("Chunk length must be superior to stride length.")
106
-
107
- for item in self.chunk_iter_with_batch(
108
- inputs,
109
- chunk_len,
110
- stride_left,
111
- stride_right,
112
- batch_size,
113
- ):
114
- yield item
115
- else:
116
- processed = self.feature_extractor(
117
- inputs, sampling_rate=self.feature_extractor.sampling_rate, return_tensors="np"
118
- )
119
- if stride is not None:
120
- processed["stride"] = stride
121
- yield processed
122
-
123
- def postprocess(self, model_outputs, return_timestamps=None, return_language=None):
124
- # unpack the outputs from list(dict(list)) to list(dict)
125
- model_outputs = [dict(zip(output, t)) for output in model_outputs for t in zip(*output.values())]
126
-
127
- time_precision = self.feature_extractor.chunk_length / 1500 # max source positions = 1500
128
- # Send the chunking back to seconds, it's easier to handle in whisper
129
- sampling_rate = self.feature_extractor.sampling_rate
130
- for output in model_outputs:
131
- if "stride" in output:
132
- chunk_len, stride_left, stride_right = output["stride"]
133
- # Go back in seconds
134
- chunk_len /= sampling_rate
135
- stride_left /= sampling_rate
136
- stride_right /= sampling_rate
137
- output["stride"] = chunk_len, stride_left, stride_right
138
-
139
- text, optional = self.tokenizer._decode_asr(
140
- model_outputs,
141
- return_timestamps=return_timestamps,
142
- return_language=return_language,
143
- time_precision=time_precision,
144
- )
145
- return {"text": text, **optional}
146
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
requirements.txt DELETED
@@ -1,4 +0,0 @@
1
- transformers
2
- pytube
3
- requests>=2.28.2
4
-
 
 
 
 
 
run.sh ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # Careful: can't create tmp files from this script
4
+ cat nginx.conf | sed "s|API_URL|${API_URL}|g" > /etc/nginx/sites-available/default
5
+ service nginx start
6
+ sleep infinity