您需要下載fonts.zip并將其解壓縮到與要運行的示例代碼相同的檔案夾中。
此代碼的目的是生成隨機文本、渲染文本并將其保存為影像。代碼接受letters和numbers分別是要從中生成文本的字母和數字的總體。它還接受character_frequency哪個決定將生成多少個字符的實體。然后生成一個長字串，并將其拆分為存盤在TextGenerator.dataset由TextGenerator.initialize_dataset.
例如：對于字母 = 'abc'，數字 = '123'，字符頻率 = 3，'aaabbbccc111222333' 被生成、打亂并拆分為隨機大小的子字串，例如：['a312c'、'1b1'、'bba32c3a2c']。
然后每個單詞將被渲染并保存為影像，TextGenerator.save_images這是這個問題的主題。
有executor將被引數concurrent.futures.ThreadPoolExecutor并concurrent.futures.ProcessPoolExecutor傳遞給TextGenerator在用于演示目的如下所示的例子。
問題是什么？
更character_frequency增加時，再存盤在資料集中TextGenerator.dataset會然而，應該不會影響性能。實際發生的：越character_frequency，更多的時間TextGenerator.save_images，需要完成與concurrent.futures.ProcessPoolExecutor。另一方面，在一切保持不變的情況下，concurrent.futures.ThreadPoolExecutor取而代之的是傳遞，所需的時間是恒定的，不受 的影響character_frequency。
import random
import string
import tempfile
import textwrap
from concurrent.futures import (ProcessPoolExecutor, ThreadPoolExecutor,
as_completed)
from pathlib import Path
from time import perf_counter
import numpy as np
import pandas as pd
from cv2 import cv2
from PIL import Image, ImageDraw, ImageFont
class TextGenerator:
def __init__(
self,
fonts,
character_frequency,
executor,
max_images=None,
background_colors=((255, 255, 255),),
font_colors=((0, 0, 0),),
font_sizes=(25,),
max_example_size=25,
min_example_size=1,
max_chars_per_line=80,
output_dir='data',
workers=1,
split_letters=False,
):
assert (
min_example_size > 0
), f'`min_example_size` should be > 0`, got {min_example_size}'
assert (
max_example_size > 0
), f'`max_example_size` should be > 0`, got {max_example_size}'
self.fonts = fonts
self.character_frequency = character_frequency
self.executor = executor
self.max_images = max_images
self.background_colors = background_colors
self.font_colors = font_colors
self.font_sizes = font_sizes
self.max_example_size = max_example_size
self.min_example_size = min_example_size
self.max_chars_per_line = max_chars_per_line
self.output_dir = Path(output_dir)
self.workers = workers
self.split_letters = split_letters
self.digits = len(f'{character_frequency}')
self.max_font = max(font_sizes)
self.generated_labels = []
self.dataset = []
self.dataset_size = 0
def render_text(self, text_lines):
font = random.choice(self.fonts)
font_size = random.choice(self.font_sizes)
background_color = random.choice(self.background_colors)
font_color = random.choice(self.font_colors)
max_width, total_height = 0, 0
font = ImageFont.truetype(font, font_size)
line_sizes = {}
for line in text_lines:
width, height = font.getsize(line)
line_sizes[line] = width, height
max_width = max(width, max_width)
total_height = height
image = Image.new('RGB', (max_width, total_height), background_color)
draw = ImageDraw.Draw(image)
current_height = 0
for line_text, dimensions in line_sizes.items():
draw.text((0, current_height), line_text, font_color, font=font)
current_height = dimensions[1]
return np.array(image)
def display_progress(self, example_idx):
print(
f'\rGenerating example {example_idx 1}/{self.dataset_size}',
end='',
)
def generate_example(self, text_lines, example_idx):
text_box = self.render_text(text_lines)
filename = (self.output_dir / f'{example_idx:0{self.digits}d}.jpg').as_posix()
cv2.imwrite(filename, text_box)
return filename, text_lines
def create_dataset_pool(self, executor, example_idx):
future_items = []
for j in range(self.workers):
if not self.dataset:
break
text = self.dataset.pop()
if text.strip():
text_lines = textwrap.wrap(text, self.max_chars_per_line)
future_items.append(
executor.submit(
self.generate_example,
text_lines,
j example_idx,
)
)
return future_items
def write_images(self):
i = 0
with self.executor(self.workers) as executor:
while i < self.dataset_size:
future_items = self.create_dataset_pool(executor, i)
for future_item in as_completed(future_items):
filename, text_lines = future_item.result()
if filename:
self.generated_labels.append(
{'filename': filename, 'label': '\n'.join(text_lines)}
)
self.display_progress(i)
i = min(self.workers, self.dataset_size - i)
if self.max_images and i >= self.max_images:
break
def initialize_dataset(self, letters, numbers, space_freq):
for characters in letters, numbers:
dataset = list(
''.join(
letter * self.character_frequency
for letter in characters ' ' * space_freq
)
)
random.shuffle(dataset)
self.dataset.extend(dataset)
i = 0
temp_dataset = []
min_split_example_size = min(self.max_example_size, self.max_chars_per_line)
total_letters = len(self.dataset)
while i < total_letters - self.min_example_size:
example_size = random.randint(self.min_example_size, self.max_example_size)
example = ''.join(self.dataset[i : i example_size])
temp_dataset.append(example)
i = example_size
if self.split_letters:
split_example = ' '.join(list(example))
for sub_example in textwrap.wrap(split_example, min_split_example_size):
if (sub_example_size := len(sub_example)) >= self.min_example_size:
temp_dataset.append(sub_example)
i = sub_example_size
self.dataset = temp_dataset
self.dataset_size = len(self.dataset)
def generate(self, letters, numbers, space_freq, fp='labels.csv'):
self.output_dir.mkdir(parents=True, exist_ok=True)
self.initialize_dataset(letters, numbers, space_freq)
t1 = perf_counter()
self.write_images()
t2 = perf_counter()
print(
f'\ntotal time: {t2 - t1} seconds, character frequency '
f'specified: {self.character_frequency}, type: {self.executor.__name__}'
)
pd.DataFrame(self.generated_labels).to_csv(self.output_dir / fp, index=False)
if __name__ == '__main__':
out = Path(tempfile.mkdtemp())
total_images = 15
for char_freq in [100, 1000, 1000000]:
for ex in [ThreadPoolExecutor, ProcessPoolExecutor]:
g = TextGenerator(
[
p.as_posix()
for p in Path('fonts').glob('*.ttf')
],
char_freq,
ex,
max_images=total_images,
output_dir=out,
max_example_size=15,
min_example_size=5,
)
g.generate(string.ascii_letters, '0123456789', 1)
在我的 i5 mbp 上產生以下結果：
Generating example 15/649
total time: 0.0652076720000001 seconds, character frequency specified: 100, type: ThreadPoolExecutor
Generating example 15/656
total time: 1.1637316500000001 seconds, character frequency specified: 100, type: ProcessPoolExecutor
Generating example 15/6442
total time: 0.06430166800000015 seconds, character frequency specified: 1000, type: ThreadPoolExecutor
Generating example 15/6395
total time: 1.2626316840000005 seconds, character frequency specified: 1000, type: ProcessPoolExecutor
Generating example 15/6399805
total time: 0.05754961300000616 seconds, character frequency specified: 1000000, type: ThreadPoolExecutor
Generating example 15/6399726
total time: 45.18768219699999 seconds, character frequency specified: 1000000, type: ProcessPoolExecutor
0.05 秒（執行緒）與 45 秒（行程）保存 15 個影像，character_frequency= 1000000。為什么要花這么長時間？為什么它會受到character_frequency價值的影響？這是獨立的，應該只影響初始化時間（這正是執行緒發生的情況）
uj5u.com熱心網友回復：
假設我正確解釋了您的代碼，您將生成大小由character_frequency值控制的示例文本。值越大，文本越長。
文本是在程式的主回圈中生成的。然后你安排一組接收所述文本并基于它生成影像的任務。
由于行程位于單獨的記憶體地址空間中，因此需要通過管道將文本發送給它們。該管道是影響您性能的瓶頸。您看到性能隨著 的增長而惡化的原因character_frequency是因為需要序列化更多文本并按順序通過所述管道發送。您的作業人員在等待資料到達時正在挨餓。
此問題不會影響您的執行緒池，因為執行緒位于主行程的相同記憶體地址空間中。因此，資料不需要序列化并跨作業系統發送。
為了在使用行程的同時加速你的程式，你可以在 worker 本身中移動文本生成邏輯，或者將所述文本寫入一個或多個檔案。然后讓作業行程自己打開這些檔案，以便您可以利用 I/O 并行化。您的主要流程所做的就是將作業人員指向正確的檔案位置或檔案名。
轉載請註明出處，本文鏈接：https://www.uj5u.com/qiye/399412.html
上一篇：如何加速我撰寫的Python代碼：包含嵌套函式的函式
下一篇：使用if陳述句向后回圈