Skip to content

Commit 424b66d

Browse files
authored
Check links in md files in parallel (#2984)
1 parent c1b20ca commit 424b66d

File tree

1 file changed

+49
-8
lines changed

1 file changed

+49
-8
lines changed

ts_scripts/sanity_utils.py

+49-8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import asyncio
12
import glob
23
import json
34
import os
@@ -19,17 +20,57 @@
1920
)
2021

2122

22-
def run_markdown_link_checker():
23-
print("## Started markdown link checker")
24-
result = True
25-
for mdfile in glob.glob("**/*.md", recursive=True):
23+
async def markdown_link_checker(in_queue, out_queue, n):
24+
print(f"worker started {n}")
25+
while True:
26+
mdfile = await in_queue.get()
27+
output = []
28+
result = True
2629
cmd = f"markdown-link-check {mdfile} --config link_check_config.json"
27-
print(f"## In directory: {os.getcwd()} | Executing command: {cmd}")
28-
status = os.system(cmd)
30+
output.append(f"## In directory: {os.getcwd()} | Executing command: {cmd}")
31+
p = await asyncio.create_subprocess_shell(
32+
cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
33+
)
34+
while not p.stdout.at_eof():
35+
line = await p.stdout.readline()
36+
output.append(line.decode("utf-8"))
37+
38+
status = await p.wait()
2939
if status != 0:
30-
print(f"## Broken links in file: {mdfile}")
40+
output.append(f"## Broken links in file: {mdfile}")
3141
result = False
32-
return result
42+
await out_queue.put((result, output))
43+
44+
45+
async def run_markdown_link_checker_on_files(files):
46+
results = []
47+
tasks = []
48+
in_queue = asyncio.Queue()
49+
out_queue = asyncio.Queue()
50+
for f in files:
51+
in_queue.put_nowait(f)
52+
53+
for n in range(16):
54+
tasks.append(asyncio.create_task(markdown_link_checker(in_queue, out_queue, n)))
55+
56+
while len(results) != len(files):
57+
print(len(results))
58+
r, output = await out_queue.get()
59+
results.append(r)
60+
for line in output:
61+
print(line)
62+
63+
for t in tasks:
64+
t.cancel()
65+
66+
return results
67+
68+
69+
def run_markdown_link_checker():
70+
print("## Started markdown link checker")
71+
files = glob.glob("**/*.md", recursive=True)
72+
results = asyncio.run(run_markdown_link_checker_on_files(files))
73+
return all(results)
3374

3475

3576
def validate_model_on_gpu():

0 commit comments

Comments
 (0)