Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,11 @@ jobs:
- name: Install
working-directory: /home/runner/frappe-bench
run: |
bench get-app wiki --branch develop
bench get-app wikify $GITHUB_WORKSPACE
bench setup requirements --dev
bench new-site --db-root-password root --admin-password admin test_site
bench --site test_site install-app wiki
bench --site test_site install-app wikify
bench build
env:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ venv.bak/
# Dependency directories
node_modules/
jspm_packages/
# `bench build` symlinks the app's node_modules into public/; git sees a symlink,
# not a directory, so the `node_modules/` rule above does not cover it.
wikify/public/node_modules

# Frontend build output (SPA). www/wikify.html is emitted by the vite plugin.
frontend/dist/
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
exclude: 'node_modules|.git'
exclude: 'node_modules|\.git|^pocs/|^scratch/'
default_stages: [pre-commit]
fail_fast: false

Expand Down
1 change: 0 additions & 1 deletion frontend/src/components/AgentChatPanel.vue
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,6 @@ function onKeydown(e) {
</Dropdown>
</div>
</div>

</aside>
</Transition>
</template>
48 changes: 38 additions & 10 deletions frontend/src/components/AppSettingsDialog.vue
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,21 @@ const newKey = ref("");

const MODEL_FIELDS = [
{ key: "agent_model", label: "Agent model", hint: "Assistant model (projects can override)." },
{ key: "vlm_model", label: "VLM model", hint: "Vision model for re-parsing flagged / visual pages." },
{ key: "cleanup_model", label: "Cleanup model", hint: "Cheap text model for the markdown cleanup pass." },
{ key: "judge_model", label: "Judge model", hint: "Independent grader — keep distinct from the parser." },
{
key: "vlm_model",
label: "VLM model",
hint: "Vision model for re-parsing flagged / visual pages.",
},
{
key: "cleanup_model",
label: "Cleanup model",
hint: "Cheap text model for the markdown cleanup pass.",
},
{
key: "judge_model",
label: "Judge model",
hint: "Independent grader — keep distinct from the parser.",
},
{ key: "classifier_model", label: "Classifier model", hint: "" },
];

Expand Down Expand Up @@ -108,7 +120,10 @@ const activeTab = ref(TABS[0].value);
<SettingsNavGroup label="Wikify">
<SettingsNavItem v-for="tab in TABS" :key="tab.value" :value="tab.value">
<template #prefix>
<span :class="[tab.icon, 'size-4 shrink-0 text-ink-gray-6']" aria-hidden="true" />
<span
:class="[tab.icon, 'size-4 shrink-0 text-ink-gray-6']"
aria-hidden="true"
/>
</template>
{{ tab.label }}
</SettingsNavItem>
Expand Down Expand Up @@ -138,14 +153,17 @@ const activeTab = ref(TABS[0].value);
:placeholder="keyIsSet ? '•••• key saved — type to replace' : 'sk-or-v1-…'"
/>
<p class="mt-1.5 text-p-sm text-ink-gray-5">
Used for all AI steps. Stored encrypted. Leave blank to keep the current key;
falls back to site config / the app .env when never set.
Used for all AI steps. Stored encrypted. Leave blank to keep the current
key; falls back to site config / the app .env when never set.
</p>
</SettingsBody>
</SettingsPanel>

<SettingsPanel value="models">
<SettingsHeader title="Models" description="OpenRouter model ids per pipeline step.">
<SettingsHeader
title="Models"
description="OpenRouter model ids per pipeline step."
>
<template #actions>
<Button
variant="solid"
Expand All @@ -167,14 +185,19 @@ const activeTab = ref(TABS[0].value);
type="text"
placeholder="e.g. anthropic/claude-sonnet-4.6"
/>
<p v-if="f.hint" class="mt-1.5 text-p-sm text-ink-gray-5">{{ f.hint }}</p>
<p v-if="f.hint" class="mt-1.5 text-p-sm text-ink-gray-5">
{{ f.hint }}
</p>
</div>
</div>
</SettingsBody>
</SettingsPanel>

<SettingsPanel value="scoring">
<SettingsHeader title="Scoring & thresholds" description="Judge gates and parse tuning.">
<SettingsHeader
title="Scoring & thresholds"
description="Judge gates and parse tuning."
>
<template #actions>
<Button
variant="solid"
Expand All @@ -196,7 +219,12 @@ const activeTab = ref(TABS[0].value);
<Switch v-model="form.judge_all_pages" />
</SettingsRow>
<SettingsRow v-for="f in FLOAT_FIELDS" :key="f.key" :title="f.label">
<FormControl v-model="form[f.key]" type="number" step="0.001" class="w-28" />
<FormControl
v-model="form[f.key]"
type="number"
step="0.001"
class="w-28"
/>
</SettingsRow>
<SettingsRow v-for="f in INT_FIELDS" :key="f.key" :title="f.label">
<FormControl v-model="form[f.key]" type="number" class="w-28" />
Expand Down
5 changes: 4 additions & 1 deletion frontend/src/components/Explore.vue
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ function loadSummary() {
}
function loadResults() {
if (props.sourceDocument && selectedType.value) {
results.submit({ section_type: selectedType.value, source_document: props.sourceDocument });
results.submit({
section_type: selectedType.value,
source_document: props.sourceDocument,
});
}
}

Expand Down
13 changes: 10 additions & 3 deletions frontend/src/components/GraphView.vue
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,15 @@ function setGraph(data) {
forceLink(links)
.id((d) => d.id)
.distance((l) => (l.rel === "REFERENCES" ? 90 : 42))
.strength((l) => (l.rel === "REFERENCES" ? Math.min(1, 0.3 + 0.1 * l.weight) : 0.5))
.strength((l) =>
l.rel === "REFERENCES" ? Math.min(1, 0.3 + 0.1 * l.weight) : 0.5
)
)
.force("charge", forceManyBody().strength(-160))
.force("collide", forceCollide().radius((d) => radius(d) + 7))
.force(
"collide",
forceCollide().radius((d) => radius(d) + 7)
)
// Gentle centering (not forceCenter): keeps disconnected components on-canvas.
.force("x", forceX(0).strength(0.06))
.force("y", forceY(0).strength(0.06))
Expand All @@ -169,7 +174,9 @@ function zoomToFit() {
const [x0, x1] = [Math.min(...xs), Math.max(...xs)];
const [y0, y1] = [Math.min(...ys), Math.max(...ys)];
const k = Math.min(2, 0.9 * Math.min(w / Math.max(1, x1 - x0), h / Math.max(1, y1 - y0)));
const t = zoomIdentity.translate(w / 2 - (k * (x0 + x1)) / 2, h / 2 - (k * (y0 + y1)) / 2).scale(k);
const t = zoomIdentity
.translate(w / 2 - (k * (x0 + x1)) / 2, h / 2 - (k * (y0 + y1)) / 2)
.scale(k);
select(canvas.value).call(zoomBehavior.transform, t);
}

Expand Down
2 changes: 1 addition & 1 deletion frontend/src/components/ProjectSettings.vue
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ watch(
archived: doc.status === "Archived",
};
},
{ immediate: true },
{ immediate: true }
);

const save = useCall({
Expand Down
3 changes: 2 additions & 1 deletion frontend/src/components/SectionTree.vue
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ watch(
}
tree.value = roots;
if (roots.length && !byName.value[selectedName.value]) {
const wanted = props.initialSection && nodes[props.initialSection] ? props.initialSection : null;
const wanted =
props.initialSection && nodes[props.initialSection] ? props.initialSection : null;
selectedName.value = wanted || roots[0].name;
if (wanted)
nextTick(() =>
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/pages/ExploreGlobal.vue
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ const matchCount = computed(() => (groups.data || []).reduce((n, g) => n + g.sec

const selectedType = ref(null);
const selected = computed(
() => types.value.find((t) => t.type_name === selectedType.value) || null,
() => types.value.find((t) => t.type_name === selectedType.value) || null
);

// Re-pull the type summary whenever the project scope changes; the selected-type watcher
Expand Down
6 changes: 3 additions & 3 deletions frontend/src/pages/ImportDetail.vue
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ watch(
(key) => {
const i = tabKeys.indexOf(key);
if (i >= 0 && i !== activeTab.value) activeTab.value = i;
},
}
);

// Streaming log
Expand All @@ -88,7 +88,7 @@ watch(
if (sd) setDocument({ name: sd, label: imp.doc?.import_title || props.name }, projectChip);
else if (projectChip) setProject(projectChip);
},
{ immediate: true },
{ immediate: true }
);

const pageReview = ref(null);
Expand All @@ -104,7 +104,7 @@ const sdStats = useList({
});
const sourceStats = computed(() => sdStats.data?.[0] || null);
const docAudit = computed(
() => sourceStats.value?.canonical_mean ?? sourceStats.value?.mean_score,
() => sourceStats.value?.canonical_mean ?? sourceStats.value?.mean_score
);
function fmtCost(v) {
return v ? `$${Number(v).toFixed(4)}` : "—";
Expand Down
4 changes: 2 additions & 2 deletions frontend/src/pages/ProjectDetail.vue
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ const project = useDoc({ doctype: "Wikify Project", name: props.name });
watch(
() => props.name,
(name) => name && setProject({ name, label: project.doc?.project_name || name }),
{ immediate: true },
{ immediate: true }
);
watch(
() => project.doc?.project_name,
(label) => label && setProject({ name: props.name, label }),
(label) => label && setProject({ name: props.name, label })
);

const showNewImport = ref(false);
Expand Down
4 changes: 2 additions & 2 deletions frontend/src/pages/ProjectList.vue
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ const projects = useList({
// default is never archivable, so it always shows).
const showArchived = ref(false);
const visibleProjects = computed(() =>
(projects.data || []).filter((p) => showArchived.value || p.status !== "Archived"),
(projects.data || []).filter((p) => showArchived.value || p.status !== "Archived")
);
const archivedCount = computed(
() => (projects.data || []).filter((p) => p.status === "Archived").length,
() => (projects.data || []).filter((p) => p.status === "Archived").length
);

const showNew = ref(false);
Expand Down
11 changes: 7 additions & 4 deletions frontend/src/utils/mermaid.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ function themeConfig() {
*/
export async function renderMermaidIn(root) {
if (!root) return;
const blocks = [
...root.querySelectorAll("code.language-mermaid, pre.mermaid"),
].filter((el) => !el.closest("[data-mermaid-done]"));
const blocks = [...root.querySelectorAll("code.language-mermaid, pre.mermaid")].filter(
(el) => !el.closest("[data-mermaid-done]")
);
if (!blocks.length) return;

let mermaid;
Expand All @@ -65,7 +65,10 @@ export async function renderMermaidIn(root) {
host.setAttribute("data-mermaid-done", "");
try {
counter += 1;
const { svg } = await mermaid.render(`wikify-mermaid-${counter}`, el.textContent || "");
const { svg } = await mermaid.render(
`wikify-mermaid-${counter}`,
el.textContent || ""
);
const figure = document.createElement("div");
figure.className = "mermaid-figure";
figure.setAttribute("data-mermaid-done", "");
Expand Down
19 changes: 19 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"private": true,
"name": "wikify",
"description": "PDFs to Frappe Wiki Spaces/Documents",
"repository": {
"type": "git",
"url": "git+https://github.com/bwhtech/wikify.git"
},
"author": "BWH",
"license": "MIT",
"bugs": {
"url": "https://github.com/bwhtech/wikify/issues"
},
"scripts": {
"postinstall": "cd frontend && yarn install",
"dev": "cd frontend && yarn dev",
"build": "cd frontend && yarn build"
}
}
4 changes: 1 addition & 3 deletions wikify/agent/tools/read.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,7 @@ def _read_wiki_page(ctx: Ctx, args: dict) -> str:
name = args.get("name")
if not name:
return _("Provide the section `name`.")
sec = frappe.db.get_value(
"Source Section", name, ["title", "markdown", "wiki_document"], as_dict=True
)
sec = frappe.db.get_value("Source Section", name, ["title", "markdown", "wiki_document"], as_dict=True)
if not sec:
return _("Section {0} not found.").format(name)
if not sec.wiki_document or not frappe.db.exists("Wiki Document", sec.wiki_document):
Expand Down
7 changes: 3 additions & 4 deletions wikify/agent/tools/reparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,9 @@ def _use_page_image(ctx: Ctx, args: dict) -> str:
embed_page_image(source_document, page_no)
except (ValueError, RuntimeError) as e:
return _("Couldn't embed the page image: {0}").format(str(e))
return (
_("Page {0} now embeds its rendered image as canonical markdown (no re-parse).").format(page_no)
+ _propagate_page(source_document, page_no)
)
return _("Page {0} now embeds its rendered image as canonical markdown (no re-parse).").format(
page_no
) + _propagate_page(source_document, page_no)


def _reparse_page(ctx: Ctx, args: dict) -> str:
Expand Down
5 changes: 4 additions & 1 deletion wikify/agent/tools/tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,10 @@ def _merge_sections(ctx: Ctx, args: dict) -> str:
"properties": {
"name": {"type": "string", "description": "Source Section id to split."},
"at_heading": {"type": "string", "description": "Heading text to split at."},
"new_title": {"type": "string", "description": "Title for the new sibling (defaults to the heading text)."},
"new_title": {
"type": "string",
"description": "Title for the new sibling (defaults to the heading text).",
},
},
"required": ["name", "at_heading"],
},
Expand Down
4 changes: 1 addition & 3 deletions wikify/api/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@

def _document_graph(source_document: str) -> tuple[list[dict], list[dict]]:
"""(nodes, edges) for one Source Document — shared by both scopes."""
sd = frappe.db.get_value(
"Source Document", source_document, ["title", "page_count"], as_dict=True
)
sd = frappe.db.get_value("Source Document", source_document, ["title", "page_count"], as_dict=True)
if not sd:
frappe.throw(_("Source Document {0} not found.").format(source_document))
sections = frappe.get_all(
Expand Down
4 changes: 1 addition & 3 deletions wikify/engine/loader/wiki.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@

_SLUG_RE = re.compile(r"[^a-z0-9]+")
# Optional see/refer cue, the word page/pg/p (with optional "no"), then the number.
_PAGEREF_RE = re.compile(
r"((?:see|refer(?:\s+to)?)\s+)?(page\s*no\.?|page|pg\.?|p\.)\s*(\d{1,4})\b", re.I
)
_PAGEREF_RE = re.compile(r"((?:see|refer(?:\s+to)?)\s+)?(page\s*no\.?|page|pg\.?|p\.)\s*(\d{1,4})\b", re.I)


def slugify(text: str) -> str:
Expand Down
14 changes: 10 additions & 4 deletions wikify/engine/remediate.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,9 @@ def remediate_pdf(
candidates: list[tuple] = [] # (method, markdown, PageScore, adopt_eligible)
errors: list[str] = []
try:
vlm_md = vlm.parse_page_image(data_url, project_context=project_context, instruction=instruction)
vlm_md = vlm.parse_page_image(
data_url, project_context=project_context, instruction=instruction
)
vlm_ps = score_page(
p["page_no"], vlm_md, gt, image_data_url=img, use_judge=use_judge, page_kind=kind
)
Expand All @@ -121,7 +123,9 @@ def remediate_pdf(
errors.append(f"vlm failed: {e}")
if kind != "visual":
try:
clean_md = clean_markdown(base_md, project_context=project_context, instruction=instruction)
clean_md = clean_markdown(
base_md, project_context=project_context, instruction=instruction
)
clean_ps = score_page(
p["page_no"], clean_md, gt, image_data_url=img, use_judge=use_judge, page_kind=kind
)
Expand All @@ -137,8 +141,10 @@ def remediate_pdf(
winner = _pick_winner(candidates)
# Record the adopted candidate; when nothing is adopted, record the vlm attempt
# (the expensive audit trail) so the review UI shows what was tried and why not.
record = winner or next((c for c in candidates if c[0] == "vlm"), None) or (
candidates[0] if candidates else None
record = (
winner
or next((c for c in candidates if c[0] == "vlm"), None)
or (candidates[0] if candidates else None)
)
if record:
method, new_md, new_ps, _ = record
Expand Down
Loading
Loading