Skip to content

Commit 96bd6e0

Browse files
committed
parallelize the wikibase-dump-filter commands
The `movies:generate_movies` command went from 4.9h down to 3h.
1 parent 0c23e4f commit 96bd6e0

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

www/lib/tasks/movies.rake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ namespace :movies do
5252
output_file = File.join(output_dir, "movies.json")
5353
claim_file = Rails.root.join("config/wikibase-dump-filter-movies-claim")
5454

55-
`cat #{input_file} | npx wikibase-dump-filter --claim #{claim_file} > #{output_file}`
55+
`cat #{input_file} | parallel --pipe --block 100M --line-buffer "npx wikibase-dump-filter --claim #{claim_file}" > #{output_file}`
5656
end
5757
end
5858

@@ -63,7 +63,7 @@ namespace :movies do
6363
input_file = File.join(output_dir, "latest-all.json")
6464
output_file = File.join(output_dir, "humans.json")
6565

66-
`cat #{input_file} | npx wikibase-dump-filter --claim P31:Q5 > #{output_file}`
66+
`cat #{input_file} | parallel --pipe --block 100M --line-buffer "npx wikibase-dump-filter --claim P31:Q5" > #{output_file}`
6767
end
6868
end
6969

0 commit comments

Comments
 (0)