|
| 1 | +import asyncio |
1 | 2 | import glob
|
2 | 3 | import json
|
3 | 4 | import os
|
|
19 | 20 | )
|
20 | 21 |
|
21 | 22 |
|
22 |
| -def run_markdown_link_checker(): |
23 |
| - print("## Started markdown link checker") |
24 |
| - result = True |
25 |
| - for mdfile in glob.glob("**/*.md", recursive=True): |
| 23 | +async def markdown_link_checker(in_queue, out_queue, n): |
| 24 | + print(f"worker started {n}") |
| 25 | + while True: |
| 26 | + mdfile = await in_queue.get() |
| 27 | + output = [] |
| 28 | + result = True |
26 | 29 | cmd = f"markdown-link-check {mdfile} --config link_check_config.json"
|
27 |
| - print(f"## In directory: {os.getcwd()} | Executing command: {cmd}") |
28 |
| - status = os.system(cmd) |
| 30 | + output.append(f"## In directory: {os.getcwd()} | Executing command: {cmd}") |
| 31 | + p = await asyncio.create_subprocess_shell( |
| 32 | + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE |
| 33 | + ) |
| 34 | + while not p.stdout.at_eof(): |
| 35 | + line = await p.stdout.readline() |
| 36 | + output.append(line.decode("utf-8")) |
| 37 | + |
| 38 | + status = await p.wait() |
29 | 39 | if status != 0:
|
30 |
| - print(f"## Broken links in file: {mdfile}") |
| 40 | + output.append(f"## Broken links in file: {mdfile}") |
31 | 41 | result = False
|
32 |
| - return result |
| 42 | + await out_queue.put((result, output)) |
| 43 | + |
| 44 | + |
| 45 | +async def run_markdown_link_checker_on_files(files): |
| 46 | + results = [] |
| 47 | + tasks = [] |
| 48 | + in_queue = asyncio.Queue() |
| 49 | + out_queue = asyncio.Queue() |
| 50 | + for f in files: |
| 51 | + in_queue.put_nowait(f) |
| 52 | + |
| 53 | + for n in range(16): |
| 54 | + tasks.append(asyncio.create_task(markdown_link_checker(in_queue, out_queue, n))) |
| 55 | + |
| 56 | + while len(results) != len(files): |
| 57 | + print(len(results)) |
| 58 | + r, output = await out_queue.get() |
| 59 | + results.append(r) |
| 60 | + for line in output: |
| 61 | + print(line) |
| 62 | + |
| 63 | + for t in tasks: |
| 64 | + t.cancel() |
| 65 | + |
| 66 | + return results |
| 67 | + |
| 68 | + |
| 69 | +def run_markdown_link_checker(): |
| 70 | + print("## Started markdown link checker") |
| 71 | + files = glob.glob("**/*.md", recursive=True) |
| 72 | + results = asyncio.run(run_markdown_link_checker_on_files(files)) |
| 73 | + return all(results) |
33 | 74 |
|
34 | 75 |
|
35 | 76 | def validate_model_on_gpu():
|
|
0 commit comments