From fca56eee22e71b75a551bfbd4f44588be2f3418e Mon Sep 17 00:00:00 2001 From: Jaime Wren Date: Wed, 25 Feb 2026 15:19:45 -0800 Subject: [PATCH] This introduces a new Dart CLI application in the `tool/` directory designed to automate the creation and validation of Agent Skills Contributions from @jwren as well as @johnpryan https://platform.claude.com/docs/en/agents-and-tools/agent-skills/overview --- .gitignore | 18 + tool/README.md | 95 ++++ tool/analysis_options.yaml | 138 +++++ tool/bin/skills.dart | 68 +++ tool/lib/src/commands/base_skill_command.dart | 114 ++++ .../src/commands/generate_skill_command.dart | 75 +++ .../src/commands/validate_skill_command.dart | 128 +++++ tool/lib/src/models/skill_params.dart | 36 ++ tool/lib/src/services/gemini_service.dart | 348 ++++++++++++ tool/lib/src/services/markdown_converter.dart | 226 ++++++++ tool/lib/src/services/skill_instructions.dart | 29 + tool/pubspec.lock | 501 +++++++++++++++++ tool/pubspec.yaml | 29 + .../fetch_and_convert_content_test.dart | 65 +++ tool/test/generate_skills_retry_test.dart | 104 ++++ tool/test/generate_skills_test.dart | 327 +++++++++++ tool/test/markdown_converter_test.dart | 103 ++++ tool/test/markdown_table_test.dart | 87 +++ tool/test/models/skill_params_test.dart | 37 ++ tool/test/services/gemini_service_test.dart | 368 +++++++++++++ .../services/markdown_converter_test.dart | 107 ++++ tool/test/skill_assertions_test.dart | 123 +++++ tool/test/validate_skills_test.dart | 515 ++++++++++++++++++ tool/test/yaml_assets_test.dart | 184 +++++++ 24 files changed, 3825 insertions(+) create mode 100644 .gitignore create mode 100644 tool/README.md create mode 100644 tool/analysis_options.yaml create mode 100644 tool/bin/skills.dart create mode 100644 tool/lib/src/commands/base_skill_command.dart create mode 100644 tool/lib/src/commands/generate_skill_command.dart create mode 100644 tool/lib/src/commands/validate_skill_command.dart create mode 100644 tool/lib/src/models/skill_params.dart create mode 100644 tool/lib/src/services/gemini_service.dart create mode 100644 tool/lib/src/services/markdown_converter.dart create mode 100644 tool/lib/src/services/skill_instructions.dart create mode 100644 tool/pubspec.lock create mode 100644 tool/pubspec.yaml create mode 100644 tool/test/commands/fetch_and_convert_content_test.dart create mode 100644 tool/test/generate_skills_retry_test.dart create mode 100644 tool/test/generate_skills_test.dart create mode 100644 tool/test/markdown_converter_test.dart create mode 100644 tool/test/markdown_table_test.dart create mode 100644 tool/test/models/skill_params_test.dart create mode 100644 tool/test/services/gemini_service_test.dart create mode 100644 tool/test/services/markdown_converter_test.dart create mode 100644 tool/test/skill_assertions_test.dart create mode 100644 tool/test/validate_skills_test.dart create mode 100644 tool/test/yaml_assets_test.dart diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f62b970a --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# IntelliJ +*.iml +*.ipr +*.iws +.idea/ + +# Mac +.DS_Store + +# Coverage +coverage/ + +# Validation output from the validate-skill command +validation/ diff --git a/tool/README.md b/tool/README.md new file mode 100644 index 00000000..84b3d8e9 --- /dev/null +++ b/tool/README.md @@ -0,0 +1,95 @@ +# Skills CLI + +The Skills CLI simplifies the process of creating "Agent Skills" from external documentation. It allows you to crawl documentation websites to discover relevant pages and then uses Generative AI (Gemini) to convert those pages into structured `SKILL.md` files that agents can use. + +## Context + +* [Agent Skills Best Practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices) + +## Prerequisite + +This tool requires the `GEMINI_API_KEY` environment variable to be set. + +## Commands + +### `generate-skill` + +Generates `SKILL.md` files from a JSON configuration file. Use the `--skill` option to generate a specific skill. + +**Usage:** +```bash +dart run skills generate-skill [options] [config_file] +``` + +**Arguments:** +* `[config_file]`: Path to the JSON configuration file. Defaults to `resources/flutter_skills.yaml`. + +**Options:** +* `--skill`: Filter to generate only the specified skill by name. +* `--directory` (`-d`): The directory to output the generated skill folder. Defaults to `../skills/`. + +### `validate-skill` + +Validates skills by re-generating and comparing with existing skills. + +**Usage:** +```bash +dart run skills validate-skill [options] [config_file] +``` + +**Arguments:** +* `[config_file]`: Path to the JSON configuration file. Defaults to `resources/flutter_skills.json`. + +**Options:** +* `--skill`: Validate only the specified skill by name. +* `--directory` (`-d`): The directory containing the generated skills. Defaults to the output directory or `../skills/`. +* `--thinking-budget`: The token budget for the model to "think" before generating content. Defaults to 2048. + +**Example:** +Generate all skills defined in resources/flutter_skills.yaml to the skills/ directory: + +```bash +dart run skills generate-skill +``` + +Generate only the 'flutter-layout' skill to a custom directory: + +``` +dart run skills generate-skill --skill flutter-layout --directory ../skills +``` + +### `validate-skill` + +Validates generated skills by re-generating them using the same source and comparing the output. This is useful for testing prompts or verifying consistency. + +**Usage:** +```bash +dart run skills validate-skill [options] +``` + +**Options:** +* `--directory` (`-d`): The directory containing the generated skills to validate. Defaults to `skills/`. + +**Example:** +Validate skills in the default 'skills' directory: + +```bash +dart run skills validate-skill +``` + +Validate skills in a custom directory: +``` +dart run skills validate-skill --directory ../validation_results +``` + +## Configuration + +The default configuration file is located at `tool/resources/flutter_skills.yaml`. It contains a list of skill definitions: + +```yaml +- name: flutter-layout + description: "..." + resources: + - https://docs.flutter.dev/ui/widgets/layout + - https://docs.flutter.dev/ui/layout +``` diff --git a/tool/analysis_options.yaml b/tool/analysis_options.yaml new file mode 100644 index 00000000..4ad56cbb --- /dev/null +++ b/tool/analysis_options.yaml @@ -0,0 +1,138 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +linter: + # The lint rules applied to this project can be customized in the + # section below to disable rules from the `package:flutter_lints/flutter.yaml` + # included above or to enable additional rules. A list of all available lints + # and their documentation is published at https://dart.dev/lints. + # + # Instead of disabling a lint rule for the entire project in the + # section below, it can also be suppressed for a single line of code + # or a specific dart file by using the `// ignore: name_of_lint` and + # `// ignore_for_file: name_of_lint` syntax on the line or in the file + # producing the lint. + rules: + # Error Prevention + avoid_catches_without_on_clauses: true + avoid_catching_errors: true + avoid_returning_this: true + avoid_void_async: true + await_only_futures: true + cancel_subscriptions: true + close_sinks: true + discarded_futures: true + literal_only_boolean_expressions: true + no_adjacent_strings_in_list: true + no_logic_in_create_state: true + only_throw_errors: true + prefer_is_empty: true + prefer_is_not_empty: true + test_types_in_equals: true + throw_in_finally: true + unawaited_futures: true + unnecessary_null_aware_assignments: true + unnecessary_null_in_if_null_operators: true + unnecessary_nullable_for_final_variable_declarations: true + use_build_context_synchronously: true + use_rethrow_when_possible: true + + # Style & Formatting + always_declare_return_types: true + always_put_required_named_parameters_first: true + avoid_escaping_inner_quotes: true + avoid_function_literals_in_foreach_calls: true + avoid_init_to_null: true + avoid_multiple_declarations_per_line: true + avoid_positional_boolean_parameters: true + avoid_private_typedef_functions: true + avoid_redundant_argument_values: true + avoid_renaming_method_parameters: true + avoid_shadowing_type_parameters: true + avoid_types_on_closure_parameters: true + avoid_unused_constructor_parameters: true + camel_case_extensions: true + cascade_invocations: true + curly_braces_in_flow_control_structures: true + directives_ordering: true + eol_at_end_of_file: true + file_names: true + library_names: true + no_leading_underscores_for_library_prefixes: true + no_leading_underscores_for_local_identifiers: true + omit_local_variable_types: true + prefer_const_constructors: true + prefer_const_constructors_in_immutables: true + prefer_const_declarations: true + prefer_const_literals_to_create_immutables: true + prefer_final_fields: true + prefer_final_in_for_each: true + prefer_final_locals: true + prefer_if_elements_to_conditional_expressions: true + prefer_interpolation_to_compose_strings: true + prefer_mixin: true + prefer_relative_imports: true + prefer_single_quotes: true + require_trailing_commas: true + sized_box_for_whitespace: true + sort_child_properties_last: true + sort_constructors_first: true + sort_unnamed_constructors_first: true + type_init_formals: true + unnecessary_await_in_return: true + unnecessary_breaks: true + unnecessary_const: true + unnecessary_constructor_name: true + unnecessary_lambdas: true + unnecessary_late: true + unnecessary_new: true + unnecessary_parenthesis: true + unnecessary_raw_strings: true + unnecessary_string_escapes: true + unnecessary_string_interpolations: true + unnecessary_this: true + use_function_type_syntax_for_parameters: true + use_string_buffers: true + use_to_and_as_if_applicable: true + + # Best Practices + annotate_redeclares: true + avoid_annotating_with_dynamic: true + avoid_bool_literals_in_conditional_expressions: true + avoid_classes_with_only_static_members: true + avoid_print: true + avoid_setters_without_getters: true + depend_on_referenced_packages: true + deprecated_consistency: true + library_private_types_in_public_api: true + prefer_asserts_in_initializer_lists: true + prefer_constructors_over_static_methods: true + + # Documentation + dangling_library_doc_comments: true + library_annotations: true + public_member_api_docs: true + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/tool/bin/skills.dart b/tool/bin/skills.dart new file mode 100644 index 00000000..446cedeb --- /dev/null +++ b/tool/bin/skills.dart @@ -0,0 +1,68 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:http/http.dart' as http; +import 'package:logging/logging.dart'; +import 'package:skills/src/commands/generate_skill_command.dart'; +import 'package:skills/src/commands/validate_skill_command.dart'; + +const String version = '0.1.0'; + +void main(List arguments) async { + final httpClient = http.Client(); + + final runner = CommandRunner('skills', 'A sample command-line application.') + ..addCommand(GenerateSkillCommand(httpClient: httpClient)) + ..addCommand(ValidateSkillCommand(httpClient: httpClient)); + + runner.argParser.addFlag( + 'version', + negatable: false, + help: 'Print the tool version.', + ); + runner.argParser.addFlag( + 'verbose', + abbr: 'v', + negatable: false, + help: 'Show additional command output.', + ); + + try { + final results = runner.parse(arguments); + if (results.flag('version')) { + stdout.writeln('skills version: $version'); + return; + } + + _configureLogging(results.flag('verbose')); + + if (results.flag('verbose')) { + Logger.root.fine('All arguments: ${results.arguments}'); + } + + await runner.run(arguments); + } on UsageException catch (e) { + stderr.writeln(e); + exit(64); + } on Exception catch (e) { + stderr.writeln('An error occurred: $e'); + exit(1); + } finally { + httpClient.close(); + } +} + +void _configureLogging(bool verbose) { + Logger.root.level = verbose ? Level.ALL : Level.INFO; + Logger.root.onRecord.listen((record) { + if (record.level >= Level.SEVERE) { + stderr.writeln(record.message); + } else { + stdout.writeln(record.message); + } + }); +} diff --git a/tool/lib/src/commands/base_skill_command.dart b/tool/lib/src/commands/base_skill_command.dart new file mode 100644 index 00000000..4503b81d --- /dev/null +++ b/tool/lib/src/commands/base_skill_command.dart @@ -0,0 +1,114 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:http/http.dart' as http; +import 'package:logging/logging.dart'; +import 'package:yaml/yaml.dart'; + +import '../models/skill_params.dart'; +import '../services/gemini_service.dart'; + +/// Base command for skill operations. +abstract class BaseSkillCommand extends Command { + /// Creates a new [BaseSkillCommand]. + BaseSkillCommand({ + required this.httpClient, + required this.logger, + this.outputDir, + }) { + argParser + ..addOption('skill', help: 'Process only the specified skill by name.') + ..addOption( + 'directory', + abbr: 'd', + help: 'The directory to output/search for skills.', + ) + ..addOption( + 'thinking-budget', + help: + 'The token budget for the model to "think". Defaults to ${GeminiService.defaultThinkingBudget} (recommended for technical documentation).', + defaultsTo: GeminiService.defaultThinkingBudget.toString(), + ); + } + + /// The HTTP client used for fetching resources. + final http.Client httpClient; + + /// The directory to output or find generated skills. + final Directory? outputDir; + + /// The logger for this command. + final Logger logger; + + @override + Future run() async { + final inputFile = argResults!.rest.isNotEmpty + ? argResults!.rest.first + : 'resources/flutter_skills.yaml'; + + final file = File(inputFile); + if (!file.existsSync()) { + logger.severe('Configuration file not found: $inputFile'); + return; + } + + final yamlContent = file.readAsStringSync(); + final yamlList = loadYaml(yamlContent) as YamlList; + final skills = yamlList + .map((e) => SkillParams.fromJson(jsonDecode(jsonEncode(e)))) + .toList(); + + final skillFilter = argResults?['skill'] as String?; + final targetSkills = skillFilter != null + ? skills.where((s) => s.name == skillFilter).toList() + : skills; + + if (targetSkills.isEmpty) { + if (skillFilter != null) { + logger.warning('No skill found with name: $skillFilter'); + } else { + logger.warning('No skills found in configuration file.'); + } + return; + } + + final apiKey = Platform.environment['GEMINI_API_KEY']; + if (apiKey == null) { + logger.severe('GEMINI_API_KEY environment variable not set.'); + return; + } + + final gemini = GeminiService(apiKey: apiKey, httpClient: httpClient); + final directoryArg = argResults?['directory'] as String?; + final outDir = directoryArg != null + ? Directory(directoryArg) + : (outputDir ?? Directory('../skills')); + + int thinkingBudget; + try { + thinkingBudget = int.parse(argResults!['thinking-budget']); + } on FormatException { + logger.warning( + 'Invalid thinking-budget: ${argResults!['thinking-budget']}. Skipping.', + ); + return; + } + + for (final skill in targetSkills) { + await runSkill(skill, gemini, outDir, thinkingBudget); + } + } + + /// Executes the command for a specific skill. + Future runSkill( + SkillParams skill, + GeminiService gemini, + Directory outputDir, + int thinkingBudget, + ); +} diff --git a/tool/lib/src/commands/generate_skill_command.dart b/tool/lib/src/commands/generate_skill_command.dart new file mode 100644 index 00000000..ef6f82c2 --- /dev/null +++ b/tool/lib/src/commands/generate_skill_command.dart @@ -0,0 +1,75 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; + +import '../models/skill_params.dart'; +import '../services/gemini_service.dart'; +import 'base_skill_command.dart'; + +/// Command to generate skills from a configuration file. +class GenerateSkillCommand extends BaseSkillCommand { + /// Creates a new [GenerateSkillCommand]. + GenerateSkillCommand({required super.httpClient, super.outputDir}) + : super(logger: Logger('GenerateSkillCommand')); + + @override + String get name => 'generate-skill'; + + @override + String get description => 'Generates skills from using yaml configuration.'; + + @override + Future runSkill( + SkillParams skill, + GeminiService gemini, + Directory outputDir, + int thinkingBudget, + ) async { + logger.info('Generating skill: ${skill.name}...'); + + try { + final combinedMarkdown = await fetchAndConvertContent( + skill.resources, + httpClient, + logger, + ); + + if (combinedMarkdown.isEmpty) { + logger.warning(' No content fetched for ${skill.name}. Skipping.'); + return; + } + + final generatedContent = await gemini.generateSkillContent( + combinedMarkdown, + skill.name, + skill.description, + instructions: skill.instructions, + urls: skill.resources, + thinkingBudget: thinkingBudget, + ); + + if (generatedContent != null && generatedContent.isNotEmpty) { + final skillDir = Directory(p.join(outputDir.path, skill.name)); + if (!skillDir.existsSync()) { + skillDir.createSync(recursive: true); + } + + File( + p.join(skillDir.path, 'SKILL.md'), + ).writeAsStringSync(generatedContent); + logger.info( + ' Generated ${p.join(outputDir.path, skill.name, 'SKILL.md')}', + ); + } else { + logger.severe(' Failed to generate content for ${skill.name}'); + } + } on Exception catch (e) { + logger.severe(' Error processing ${skill.name}: $e'); + } + } +} diff --git a/tool/lib/src/commands/validate_skill_command.dart b/tool/lib/src/commands/validate_skill_command.dart new file mode 100644 index 00000000..cff7c62a --- /dev/null +++ b/tool/lib/src/commands/validate_skill_command.dart @@ -0,0 +1,128 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; + +import '../models/skill_params.dart'; +import '../services/gemini_service.dart'; +import 'base_skill_command.dart'; + +/// Command to validate skills by re-generating and comparing with existing skills. +class ValidateSkillCommand extends BaseSkillCommand { + /// Creates a new [ValidateSkillCommand]. + ValidateSkillCommand({ + required super.httpClient, + super.outputDir, + this.validationDir, + }) : super(logger: Logger('ValidateSkillCommand')); + + /// The directory to output validation reports. + final Directory? validationDir; + + @override + String get name => 'validate-skill'; + + @override + String get description => + 'Validates skills using existing skill files and yaml configuration.'; + + @override + Future runSkill( + SkillParams skill, + GeminiService gemini, + Directory outputDir, + int thinkingBudget, + ) async { + logger.info('Validating skill: ${skill.name}...'); + + try { + // Re-generate markdown content + final markdown = await fetchAndConvertContent( + skill.resources, + httpClient, + logger, + ); + + if (markdown.isEmpty) { + logger.warning( + ' No content fetched for ${skill.name}. Skipping validation.', + ); + return; + } + + // Read existing content + final existingSkillFile = File( + p.join(outputDir.path, skill.name, 'SKILL.md'), + ); + if (!existingSkillFile.existsSync()) { + logger.warning( + ' Existing skill file not found at ${existingSkillFile.path}', + ); + return; + } + + final existingSkillFileContent = existingSkillFile.readAsStringSync(); + + // Check for verbatim name and description + if (!existingSkillFileContent.contains('name: ${skill.name}')) { + logger.severe( + ' Validation Failed: Skill name mismatch in ${existingSkillFile.path}. ' + 'Expected "name: ${skill.name}"', + ); + } + + // Extract metadata from existing content + final generationDate = + RegExp( + 'last_modified: (.*)', + ).firstMatch(existingSkillFileContent)?.group(1) ?? + 'Unknown'; + final modelName = + RegExp( + 'model: (.*)', + ).firstMatch(existingSkillFileContent)?.group(1) ?? + 'Unknown'; + + // Compare + logger.info(' Comparing versions...'); + final result = await gemini.validateExistingSkillContent( + markdown, + skill.name, + skill.instructions ?? 'No instructions provided', + generationDate, + modelName, + existingSkillFileContent, + thinkingBudget: thinkingBudget, + ); + + if (result != null) { + final valDirBase = validationDir ?? Directory('validation'); + final valDir = Directory(p.join(valDirBase.path, skill.name)); + if (!valDir.existsSync()) { + valDir.createSync(recursive: true); + } + + File(p.join(valDir.path, 'validation.md')).writeAsStringSync(result); + + // Extract and log the grade + final gradeMatch = RegExp(r'Grade:\s*(\d+)').firstMatch(result); + final grade = gradeMatch?.group(1); + + logger.info( + ' Validation report written to ${p.join(valDir.path, 'validation.md')} ' + '${grade != null ? '(Grade: $grade)' : ''}', + ); + } else { + logger.severe( + ' Failed to generate validation report for ${skill.name}', + ); + } + } on Exception catch (e) { + logger.severe(' Error validating ${skill.name}: $e'); + } + } +} diff --git a/tool/lib/src/models/skill_params.dart b/tool/lib/src/models/skill_params.dart new file mode 100644 index 00000000..26888ee2 --- /dev/null +++ b/tool/lib/src/models/skill_params.dart @@ -0,0 +1,36 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Parameters for generating a skill from a URL. +class SkillParams { + /// Creates a new [SkillParams] instance. + SkillParams({ + required this.name, + required this.description, + required this.resources, + this.instructions, + }); + + /// Creates a [SkillParams] instance from a JSON map. + factory SkillParams.fromJson(Map json) { + return SkillParams( + name: json['name'] as String, + description: json['description'] as String, + resources: (json['resources'] as List).cast(), + instructions: json['instructions'] as String?, + ); + } + + /// The name of the skill. + final String name; + + /// The description of the skill. + final String description; + + /// Optional instructions for generating the skill. + final String? instructions; + + /// The resources/URLs to fetch content from. + final List resources; +} diff --git a/tool/lib/src/services/gemini_service.dart b/tool/lib/src/services/gemini_service.dart new file mode 100644 index 00000000..47dc2ed3 --- /dev/null +++ b/tool/lib/src/services/gemini_service.dart @@ -0,0 +1,348 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:google_cloud_ai_generativelanguage_v1beta/generativelanguage.dart'; +import 'package:http/http.dart' as http; +import 'package:logging/logging.dart'; +import 'package:meta/meta.dart'; +import 'package:retry/retry.dart'; +import 'package:yaml_writer/yaml_writer.dart'; + +import 'markdown_converter.dart'; +import 'skill_instructions.dart'; + +/// Service for interacting with the Gemini API to generate and validate skills. +class GeminiService { + /// Creates a new [GeminiService]. + GeminiService({ + required String apiKey, + http.Client? httpClient, + String? model, + }) : _model = model ?? defaultModel, + _client = _ApiKeyClient(httpClient ?? http.Client(), apiKey); + + /// 0.2 is a good temperature for technical material. + static const double defaultTemperature = 0.2; + + /// The default model to use for generation. + static const String defaultModel = 'models/gemini-3.1-pro-preview'; + + /// The default token budget for thinking. + static const int defaultThinkingBudget = 4096; + + /// The default max output tokens for generation. + static const int defaultMaxOutputTokens = 8192; + + /// The default safety settings to use for generation. + static final List defaultSafetySettings = [ + SafetySetting( + category: HarmCategory.harmCategoryDangerousContent, + threshold: SafetySetting_HarmBlockThreshold.blockOnlyHigh, + ), + SafetySetting( + category: HarmCategory.harmCategoryHateSpeech, + threshold: SafetySetting_HarmBlockThreshold.blockOnlyHigh, + ), + SafetySetting( + category: HarmCategory.harmCategoryHarassment, + threshold: SafetySetting_HarmBlockThreshold.blockOnlyHigh, + ), + SafetySetting( + category: HarmCategory.harmCategorySexuallyExplicit, + threshold: SafetySetting_HarmBlockThreshold.blockOnlyHigh, + ), + ]; + + final String _model; + final http.Client _client; + final Logger _logger = Logger('GeminiService'); + + /// Generates the content for a skill based on raw markdown input. + Future generateSkillContent( + String rawMarkdown, + String skillName, + String description, { + String? instructions, + List urls = const [], + int thinkingBudget = defaultThinkingBudget, + }) async { + final service = GenerativeService(client: _client); + final lastModified = HttpDate.format(DateTime.now()); + final prompt = _createSkillPrompt(rawMarkdown, instructions); + + final request = _createRequest( + prompt, + systemInstruction: skillInstructions, + thinkingBudget: thinkingBudget, + ); + + _logger.info( + ' Model: $_model, Max Output Tokens: $defaultMaxOutputTokens, Thinking Budget: $thinkingBudget', + ); + + try { + const r = RetryOptions(maxAttempts: 3); + final response = await r.retry(() async { + final res = await service.generateContent(request); + final text = res.candidates.first.content?.parts + .where((part) => !part.thought) + .map((part) => part.text) + .where((text) => text != null) + .join('\n'); + + if (text == null || text.isEmpty) { + throw const FormatException('Empty response from Gemini'); + } + + // Check for URLs in the content + // This regex matches http://, https://, or www. + final urlPattern = RegExp(r'(https?:\/\/[^\s]+)|(www\.[^\s]+)'); + if (urlPattern.hasMatch(text)) { + throw const FormatException( + 'Generated content contains URLs, which is not allowed.', + ); + } + + return text; + }, onRetry: (e) => _logger.warning('Retrying Gemini generation: $e')); + + final content = response; + + final frontMatterMap = { + 'name': skillName, + 'description': description, + 'metadata': { + 'urls': urls, + 'model': _model, + 'last_modified': lastModified, + }, + }; + + final frontmatter = '---\n${YamlWriter().write(frontMatterMap)}\n---\n'; + + return frontmatter + (cleanContent(content) ?? ''); + } on Object catch (e) { + _logger.severe('Gemini generation failed: $e'); + return null; + } + } + + /// Validate Existing Skill + Future validateExistingSkillContent( + String markdown, + String skillName, + String instructions, + String generationDate, + String modelName, + String currentSkillContent, { + int thinkingBudget = defaultThinkingBudget, + }) async { + final service = GenerativeService(client: _client); + final validationPrompt = + ''' +Validate the following skill document against the provided source material and verify if it is valid. +Focus on: +1. Accuracy: Does the skill capture the technical details correctly based on the Source Material? +2. Structure: Is the skill well-structured according to skill best practices? +3. Completeness: Is any critical information missing in the skill that is present in the Source Material? + +Context: +- The skill was originally generated on: $generationDate +- The current evaluation is using model: $modelName +- The instructions used to generate the skill were: +$instructions + +Source Material: +$markdown + +Current Skill Content: + "$currentSkillContent" +--- + +Grade the current output based on the instructions and the comparison to current website content and instructions today. +Establish a conclusion on whether the new skill is valid or not. +Reasons for a good or bad quality grade should be provided including concepts such as missing content, different model used, more than a few months old, etc. +On the very last line, output "Grade: [0-100]" representing overall quality of the skill compared to the assumed value if it were generated again today. +'''; + + final request = _createRequest( + validationPrompt, + systemInstruction: skillInstructions, + thinkingBudget: thinkingBudget, + ); + + _logger.info( + ' Model: $_model, Max Output Tokens: $defaultMaxOutputTokens, Thinking Budget: $thinkingBudget', + ); + + try { + const r = RetryOptions(maxAttempts: 3); + final response = await r.retry(() async { + final res = await service.generateContent(request); + final text = res.candidates.first.content?.parts + .where((part) => !part.thought) + .map((part) => part.text) + .where((text) => text != null) + .join('\n'); + + if (text == null || text.isEmpty) { + throw const FormatException('Empty response from Gemini'); + } + + // Check for URLs in the content + // This regex matches http://, https://, or www. + final urlPattern = RegExp(r'(https?:\/\/[^\s]+)|(www\.[^\s]+)'); + if (urlPattern.hasMatch(text)) { + throw const FormatException( + 'Generated content contains URLs, which is not allowed.', + ); + } + + return text; + }, onRetry: (e) => _logger.warning('Retrying Gemini validation: $e')); + + return response; + } on Object catch (e) { + _logger.severe('Gemini validation failed: $e'); + return null; + } + } + + /// Cleans the generated content by removing markdown code blocks and frontmatter. + @visibleForTesting + String? cleanContent(String? content) { + if (content == null) return null; + var cleaned = content; + final startMatch = RegExp( + r'^\s*```[a-zA-Z]*\s*\n', + caseSensitive: false, + ).firstMatch(cleaned); + if (startMatch != null) { + cleaned = cleaned.substring(startMatch.end); + // Remove the last triple backticks if they exist + cleaned = cleaned.replaceAll(RegExp(r'\n```\s*$'), ''); + } + + final yamlStartIndex = cleaned.indexOf('---'); + if (yamlStartIndex == 0) { + // Possible frontmatter, skip it + final end = cleaned.indexOf('---', 3); + if (end != -1) { + cleaned = cleaned.substring(end + 3).trim(); + } + } else if (yamlStartIndex > 0) { + // Maybe noise before frontmatter, try to strip it if it looks like frontmatter + final end = cleaned.indexOf('---', yamlStartIndex + 3); + if (end != -1) { + cleaned = cleaned.substring(end + 3).trim(); + } + } + + // Ensure one trailing newline + return '${cleaned.trim()}\n'; + } + + String _createSkillPrompt(String markdown, String? instructions) { + return ''' +Rewrite the following technical documentation into a high-quality "SKILL.md" file. + +DO NOT include any YAML frontmatter. Start immediately with the markdown content (e.g. headers). + +**Guidelines:** +1. **Ignore Noise**: Exclude navigation bars, footers, "Edit this page" links, and other non-technical content. +2. **Decision Trees**: If the content describes a process with multiple choices or steps, YOU MUST create a "Decision Logic" or "Flowchart" section to guide the agent. +3. **Clarity**: Use clear headings, bullet points, and code blocks. +4. **Format**: Do NOT wrap the entire output in a markdown code block (like ```markdown ... ```). Return raw markdown text. +5. **No URLs**: The content must NOT include any URLs or links. External references should be described in text only. +${instructions != null && instructions.isNotEmpty ? '6. **Special Instructions**: $instructions' : ''} + +Raw Content: +$markdown +'''; + } + + GenerateContentRequest _createRequest( + String prompt, { + String? systemInstruction, + int thinkingBudget = defaultThinkingBudget, + }) { + return GenerateContentRequest( + model: _model, + systemInstruction: systemInstruction != null + ? Content(parts: [Part(text: systemInstruction)]) + : null, + contents: [ + Content(parts: [Part(text: prompt)]), + ], + // See [GenerationConfig] in package:google_cloud_ai_generativelanguage_v1beta + generationConfig: GenerationConfig( + temperature: defaultTemperature, + maxOutputTokens: defaultMaxOutputTokens, + thinkingConfig: thinkingBudget > 0 + ? ThinkingConfig( + includeThoughts: true, + thinkingBudget: thinkingBudget, + ) + : null, + ), + safetySettings: defaultSafetySettings, + ); + } +} + +/// Result of a skill validation. +class ValidationResult { + /// Creates a new [ValidationResult]. + ValidationResult(this.report, this.score); + + /// The markdown validation report. + final String report; + + /// The similarity score (0-100). + final int score; +} + +class _ApiKeyClient extends http.BaseClient { + _ApiKeyClient(this._inner, this._apiKey); + + final http.Client _inner; + final String _apiKey; + + @override + Future send(http.BaseRequest request) { + request.headers['x-goog-api-key'] = _apiKey; + return _inner.send(request); + } +} + +/// Fetches and converts content from a list of URLs. +/// +/// Throws an [Exception] if fetching any URL fails. This strict behavior +/// prevents wasting Gemini tokens on generating low-quality skills when +/// source material is missing. +Future fetchAndConvertContent( + List urls, + http.Client httpClient, + Logger logger, +) async { + final converter = MarkdownConverter(); + final sb = StringBuffer(); + for (final url in urls) { + logger.info(' Fetching $url...'); + final response = await httpClient.get(Uri.parse(url)); + if (response.statusCode == 200) { + sb + ..writeln('--- Raw content from $url ---') + ..writeln(converter.convert(response.body)); + } else { + throw Exception( + 'Failed to fetch $url: HTTP ${response.statusCode}. ' + 'Failing fast to save Gemini tokens.', + ); + } + } + return sb.toString(); +} diff --git a/tool/lib/src/services/markdown_converter.dart b/tool/lib/src/services/markdown_converter.dart new file mode 100644 index 00000000..775e1dd5 --- /dev/null +++ b/tool/lib/src/services/markdown_converter.dart @@ -0,0 +1,226 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:html/dom.dart'; +import 'package:html/parser.dart'; + +/// Converts HTML content to Markdown. +class MarkdownConverter { + /// Converts HTML content to Markdown. + String convert(String htmlContent) { + final document = parse(htmlContent); + final body = document.body; + if (body == null) return ''; + return _convertElement(body).trim(); + } + + String _convertElement(Element element) { + final buffer = StringBuffer(); + + for (final node in element.nodes) { + if (node is Text) { + buffer.write(node.text); + } else if (node is Element) { + buffer.write(_processTag(node)); + } + } + + return buffer.toString(); + } + + String _processTag(Element element) { + final content = _convertElement(element); + + switch (element.localName) { + case 'h1': + return '\n# $content\n\n'; + case 'h2': + return '\n## $content\n\n'; + case 'h3': + return '\n### $content\n\n'; + case 'h4': + return '\n#### $content\n\n'; + case 'h5': + return '\n##### $content\n\n'; + case 'h6': + return '\n###### $content\n\n'; + case 'p': + return '$content\n\n'; + case 'a': + final href = element.attributes['href']; + return '[$content]($href)'; + case 'strong': + case 'b': + return '**$content**'; + case 'em': + case 'i': + return '*$content*'; + case 'del': + case 's': + case 'strike': + return '~~$content~~'; + case 'code': + return '`$content`'; + case 'pre': + return '\n```\n${element.text}\n```\n\n'; + case 'blockquote': + return '\n> $content\n\n'; + case 'hr': + return '\n---\n\n'; + case 'ul': + return '\n$content\n'; + case 'ol': + return '\n$content\n'; + case 'li': + return '- $content\n'; + case 'img': + final src = element.attributes['src'] ?? ''; + final alt = element.attributes['alt'] ?? ''; + return '![$alt]($src)'; + case 'video': + final src = element.attributes['src'] ?? ''; + final poster = element.attributes['poster'] ?? ''; + final title = element.attributes['title'] ?? 'Video'; + + String? videoUrl; + if (src.isNotEmpty) { + videoUrl = src; + } else { + // Fallback for source elements + videoUrl = element.children + .where((e) => e.localName == 'source') + .map((e) => e.attributes['src']) + .firstWhere((s) => s != null, orElse: () => null); + } + + if (videoUrl != null) { + if (poster.isNotEmpty) { + return '[![$title]($poster)]($videoUrl)'; + } + return '[$title]($videoUrl)'; + } + return ''; + case 'iframe': + final src = element.attributes['src'] ?? ''; + final title = element.attributes['title'] ?? 'Iframe'; + if (src.isNotEmpty) { + return '[$title]($src)'; + } + return ''; + case 'table': + return _processTable(element); + case 'dl': + return _processDefinitionList(element); + case 'dt': + return '\n**$content**\n'; + case 'dd': + return ': $content\n'; + case 'details': + // Preserve details as HTML, but convert children to markdown? + // Or just preserve the tag structure and convert internal content. + // Let's try to preserve the tag but convert content. + return '\n
\n$content\n
\n'; + case 'summary': + return '$content'; + case 'br': + return '\n'; + case 'div': + case 'section': + case 'main': + case 'article': + return '$content\n'; + default: + return content; + } + } + + String _processTable(Element table) { + // Simple table converter + // 1. Find headers (th) + // 2. Find rows (tr) + // 3. Construct markdown table + + final rows = table.querySelectorAll('tr'); + if (rows.isEmpty) return ''; + + final buffer = StringBuffer('\n'); + var headerCells = []; + final bodyRows = []; + + // Try to find thead + final thead = table.querySelector('thead'); + if (thead != null) { + final headerRow = thead.querySelector('tr'); + if (headerRow != null) { + headerCells = headerRow.querySelectorAll('th'); + if (headerCells.isEmpty) { + headerCells = headerRow.querySelectorAll('td'); + } + } + } + + // Try to find tbody + final tbody = table.querySelector('tbody'); + if (tbody != null) { + bodyRows.addAll(tbody.querySelectorAll('tr')); + } else { + // No tbody, check direct children + final allRows = table.querySelectorAll('tr'); + for (final row in allRows) { + if (thead != null && thead.contains(row)) continue; + if (!bodyRows.contains(row)) { + bodyRows.add(row); + } + } + } + + // Promote first row to header if needed + if (headerCells.isEmpty && bodyRows.isNotEmpty) { + final firstRow = bodyRows.first; + headerCells = firstRow.querySelectorAll('th'); + if (headerCells.isEmpty) { + headerCells = firstRow.querySelectorAll('td'); + } + if (bodyRows.isNotEmpty) bodyRows.removeAt(0); + } + + if (headerCells.isEmpty && bodyRows.isEmpty) return ''; + + // Write Header + buffer.write('|'); + for (final cell in headerCells) { + buffer.write(' ${_convertElement(cell).trim()} |'); + } + buffer.write('\n|'); + for (var i = 0; i < headerCells.length; i++) { + buffer.write('---|'); + } + buffer.write('\n'); + + // Write Body + for (final row in bodyRows) { + final cells = row.children + .where((e) => e.localName == 'td' || e.localName == 'th') + .toList(); + if (cells.isEmpty) continue; + buffer.write('|'); + for (final cell in cells) { + buffer.write(' ${_convertElement(cell).trim()} |'); + } + buffer.write('\n'); + } + buffer.write('\n'); + + return buffer.toString(); + } + + String _processDefinitionList(Element dl) { + final buffer = StringBuffer('\n'); + for (final child in dl.children) { + buffer.write(_processTag(child)); + } + buffer.write('\n\n'); + return buffer.toString(); + } +} diff --git a/tool/lib/src/services/skill_instructions.dart b/tool/lib/src/services/skill_instructions.dart new file mode 100644 index 00000000..1b7ce014 --- /dev/null +++ b/tool/lib/src/services/skill_instructions.dart @@ -0,0 +1,29 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Instructions for authoring Skills. +const String skillInstructions = ''' +Act as an Expert Skill Author for Gemini. Your goal is to generate a high-quality "Skill" module (a structured set of instructions and code assets) based on a user's requirements. + +Follow these strict guidelines: + +### 1. Single-File Output & Writing Style +- **One File Only:** Do not use supplementary files, external resources, or progressive disclosure. The entire skill must be contained within a single `SKILL.md` output. +- **Assume Competence:** Assume Gemini is already highly capable. Do not explain general concepts; focus strictly on specific logic, APIs, and constraints. +- **Naming & Description:** Use a concise, lowercase-and-hyphens name (e.g., `spreadsheet-automation`). For the description, use a maximum of 1024 characters and write in the THIRD PERSON (e.g., "Analyzes financial data..." not "I can analyze..."). + +### 2. Required Structure +Your output must exactly match the following structure and heading format: + +1. **# [Skill Name Title]:** A human-readable H1 title. +2. **## Goal:** A brief paragraph explaining the end state of the skill and any assumptions made about the user's environment. +3. **## Instructions:** A sequentially numbered list of steps. +4. **## Constraints:** A bulleted list of strict rules, cleanup tasks, or assumptions to avoid. + +### 3. Workflow, Code, & Reliability +- **Heavy Code Examples:** You MUST include plenty of code examples. Whenever a step requires an implementation, API call, or configuration change, provide the exact code block required. +- **Degrees of Freedom:** Use high-level instructions for reasoning tasks, but strict, immutable code blocks for fragile operations (e.g., file system changes, routing, or state management). +- **Interactive Checkpoints:** If a step requires user preference or context not usually available, use bolded text to instruct the AI to pause (e.g., "**STOP AND ASK THE USER:**"). +- **Feedback Loops:** Implement a "Validate-and-Fix" pattern where appropriate, instructing Gemini to verify its output or handle specific error states. +'''; diff --git a/tool/pubspec.lock b/tool/pubspec.lock new file mode 100644 index 00000000..331b94f1 --- /dev/null +++ b/tool/pubspec.lock @@ -0,0 +1,501 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "3b19a47f6ea7c2632760777c78174f47f6aec1e05f0cd611380d4593b8af1dbc" + url: "https://pub.dev" + source: hosted + version: "96.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "0c516bc4ad36a1a75759e54d5047cb9d15cded4459df01aa35a0b5ec7db2c2a0" + url: "https://pub.dev" + source: hosted + version: "10.2.0" + args: + dependency: "direct main" + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: "direct dev" + description: + name: coverage + sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" + url: "https://pub.dev" + source: hosted + version: "1.15.0" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + csslib: + dependency: transitive + description: + name: csslib + sha256: "09bad715f418841f976c77db72d5398dc1253c21fb9c0c7f0b0b985860b2d58e" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + file: + dependency: "direct main" + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: "direct main" + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + google_cloud: + dependency: transitive + description: + name: google_cloud + sha256: "76a1552170210c8a51500d555e93b526f92ab2a86aadbebd8b8d5f3d83a97c29" + url: "https://pub.dev" + source: hosted + version: "0.3.0" + google_cloud_ai_generativelanguage_v1beta: + dependency: "direct main" + description: + name: google_cloud_ai_generativelanguage_v1beta + sha256: "846770122f0dd98226892d00865a72d237c6ae11e069d2ba67283e60828df4f8" + url: "https://pub.dev" + source: hosted + version: "0.4.0" + google_cloud_longrunning: + dependency: transitive + description: + name: google_cloud_longrunning + sha256: "2b1e82d3e5b7a5a8880cab0e17ee7399720cf418d9938a644acb3ee2d24ef142" + url: "https://pub.dev" + source: hosted + version: "0.4.0" + google_cloud_protobuf: + dependency: transitive + description: + name: google_cloud_protobuf + sha256: cadadac845ad26f874363e3a002099d93b14bde9159bf974e537c47df86845e7 + url: "https://pub.dev" + source: hosted + version: "0.4.0" + google_cloud_rpc: + dependency: transitive + description: + name: google_cloud_rpc + sha256: "5eedf63d438db69c31b3db52f1ee5081df269c80388234e6873fac34ae57b0d2" + url: "https://pub.dev" + source: hosted + version: "0.4.0" + google_cloud_type: + dependency: transitive + description: + name: google_cloud_type + sha256: f5e8d649b7a99815f7c6c352597099858e3dbd041435d6567d871886944771a2 + url: "https://pub.dev" + source: hosted + version: "0.4.0" + google_identity_services_web: + dependency: transitive + description: + name: google_identity_services_web + sha256: "5d187c46dc59e02646e10fe82665fc3884a9b71bc1c90c2b8b749316d33ee454" + url: "https://pub.dev" + source: hosted + version: "0.3.3+1" + googleapis_auth: + dependency: "direct main" + description: + name: googleapis_auth + sha256: "3d84a0c47f98cc9bc59a6b8fc0ffd82c0fc803ead3971e4a2af943bd166953ae" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + html: + dependency: "direct main" + description: + name: html + sha256: "6d1264f2dffa1b1101c25a91dff0dc2daee4c18e87cd8538729773c073dbf602" + url: "https://pub.dev" + source: hosted + version: "0.15.6" + http: + dependency: "direct main" + description: + name: http + sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412" + url: "https://pub.dev" + source: hosted + version: "1.6.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.dev" + source: hosted + version: "6.1.0" + logging: + dependency: "direct main" + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" + url: "https://pub.dev" + source: hosted + version: "0.12.18" + meta: + dependency: "direct main" + description: + name: meta + sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" + url: "https://pub.dev" + source: hosted + version: "1.18.1" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + platform: + dependency: "direct main" + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.dev" + source: hosted + version: "3.1.6" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + retry: + dependency: "direct main" + description: + name: retry + sha256: "822e118d5b3aafed083109c72d5f484c6dc66707885e07c0fbcb8b986bba7efc" + url: "https://pub.dev" + source: hosted + version: "3.1.2" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" + url: "https://pub.dev" + source: hosted + version: "1.29.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" + url: "https://pub.dev" + source: hosted + version: "0.7.9" + test_core: + dependency: transitive + description: + name: test_core + sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" + url: "https://pub.dev" + source: hosted + version: "0.6.15" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.dev" + source: hosted + version: "15.0.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: "direct main" + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" + yaml_writer: + dependency: "direct main" + description: + name: yaml_writer + sha256: "69651cd7238411179ac32079937d4aa9a2970150d6b2ae2c6fe6de09402a5dc5" + url: "https://pub.dev" + source: hosted + version: "2.1.0" +sdks: + dart: ">=3.10.8 <4.0.0" diff --git a/tool/pubspec.yaml b/tool/pubspec.yaml new file mode 100644 index 00000000..19a4a501 --- /dev/null +++ b/tool/pubspec.yaml @@ -0,0 +1,29 @@ +name: skills +description: A command-line application for creating and validating Agent Skills. +version: 0.0.1 + +environment: + sdk: ^3.10.8 + +# Add regular dependencies here. +dependencies: + args: ^2.7.0 + file: ^7.0.1 + glob: ^2.1.3 + google_cloud_ai_generativelanguage_v1beta: ^0.4.0 + googleapis_auth: ^2.0.0 + html: ^0.15.6 + http: ^1.6.0 + logging: ^1.3.0 + meta: ^1.18.1 + path: ^1.9.1 + platform: ^3.1.6 + retry: ^3.1.2 + yaml: ^3.1.2 + yaml_writer: ^2.1.0 + +dev_dependencies: + coverage: ^1.15.0 + lints: ^6.0.0 + test: ^1.25.6 + diff --git a/tool/test/commands/fetch_and_convert_content_test.dart b/tool/test/commands/fetch_and_convert_content_test.dart new file mode 100644 index 00000000..3ada7090 --- /dev/null +++ b/tool/test/commands/fetch_and_convert_content_test.dart @@ -0,0 +1,65 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:logging/logging.dart'; +import 'package:skills/src/services/gemini_service.dart'; + +import 'package:test/test.dart'; + +void main() { + group('fetchAndConvertContent', () { + late Logger logger; + late List logs; + + setUp(() { + logger = Logger('test'); + logs = []; + Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + }); + + test('fetches and converts content successfully', () async { + final client = MockClient((request) async { + if (request.url.toString() == 'https://example.com') { + return http.Response('

Hello

', 200); + } + return http.Response('Not Found', 404); + }); + + final result = await fetchAndConvertContent( + ['https://example.com'], + client, + logger, + ); + + expect(result, contains('--- Raw content from https://example.com ---')); + expect(result, contains('# Hello')); + }); + + test('handles failed fetch', () async { + final client = MockClient((request) async { + return http.Response('Not Found', 404); + }); + + expect( + () => fetchAndConvertContent(['https://example.com'], client, logger), + throwsA(isA()), + ); + }); + + test('handles exception during fetch', () async { + final client = MockClient((request) async { + throw Exception('Network error'); + }); + + expect( + () => fetchAndConvertContent(['https://example.com'], client, logger), + throwsA(isA()), + ); + }); + }); +} diff --git a/tool/test/generate_skills_retry_test.dart b/tool/test/generate_skills_retry_test.dart new file mode 100644 index 00000000..430616b0 --- /dev/null +++ b/tool/test/generate_skills_retry_test.dart @@ -0,0 +1,104 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:skills/src/commands/generate_skill_command.dart'; +import 'package:test/test.dart'; + +void main() { + group('GenerateSkillsCommand Retry Logic', () { + late CommandRunner runner; + late Directory tempDir; + late File inputFile; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('skills_retry_test'); + inputFile = File(p.join(tempDir.path, 'input.json')); + runner = CommandRunner('skills', 'Test runner'); + }); + + tearDown(() async { + await tempDir.delete(recursive: true); + }); + + test('retries failed LLM calls up to 3 times', () async { + const url = 'https://example.com/retry'; + inputFile.writeAsStringSync( + jsonEncode([ + { + 'name': 'retry_skill', + 'description': 'Retry Description', + 'resources': [url], + }, + ]), + ); + + var attemptCount = 0; + final logs = []; + final sub = Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + addTearDown(sub.cancel); + + final mockClient = MockClient((request) async { + if (request.url.toString() == url) { + return http.Response('Content', 200); + } + + if (request.url.toString().contains('generativelanguage')) { + attemptCount++; + if (attemptCount < 3) { + throw Exception('Simulated Network Error'); + } + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': '---\nname: skill\n---\nContent'}, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Not Found', 404); + }); + + final command = GenerateSkillCommand( + httpClient: mockClient, + outputDir: tempDir, + ); + runner.addCommand(command); + + await runner.run(['generate-skill', inputFile.path]); + + expect( + attemptCount, + 3, + reason: 'Should attempt 3 times (1 initial + 2 retries)', + ); + expect( + logs, + contains(contains('Retrying Gemini generation')), + reason: 'Should log retry warnings', + ); + expect( + logs, + contains(contains('Generated')), + reason: 'Should eventually succeed', + ); + }); + }); +} diff --git a/tool/test/generate_skills_test.dart b/tool/test/generate_skills_test.dart new file mode 100644 index 00000000..f8df6972 --- /dev/null +++ b/tool/test/generate_skills_test.dart @@ -0,0 +1,327 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:skills/src/commands/generate_skill_command.dart'; +import 'package:test/test.dart'; + +void main() { + group('GenerateSkillsCommand', () { + late CommandRunner runner; + late Directory tempDir; + late File videoFile; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('skills_gen_test'); + videoFile = File(p.join(tempDir.path, 'input.json')); + runner = CommandRunner('skills', 'Test runner'); + }); + + tearDown(() async { + await tempDir.delete(recursive: true); + }); + + test('generates skill from JSON input with dart-docs- prefix', () async { + // Create input JSON in a file named dart_dev.json to trigger prefixing + videoFile = File(p.join(tempDir.path, 'dart_dev.json')); + final inputData = [ + { + 'name': 'foo', + 'description': 'Foo description', + 'resources': ['https://example.com/foo.html'], + }, + { + 'name': 'example', + 'description': 'Example description', + 'resources': ['https://example.com/'], + }, + ]; + videoFile.writeAsStringSync(jsonEncode(inputData)); + + final geminiRequests = []; + // Mock HTTP Client + final mockClient = MockClient((request) async { + final url = request.url.toString(); + + // 1. Mock content fetch + if (url.startsWith('https://example.com')) { + return http.Response( + '

Skill

Content for $url

', + 200, + ); + } + + // 2. Mock Gemini API + if (url.contains('generativelanguage.googleapis.com')) { + geminiRequests.add(request.body); + // ... strict mock ... + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Generated Content'}, + ], + }, + }, + ], + }), + 200, + ); + } + + return http.Response('Not Found', 404); + }); + + final command = GenerateSkillCommand( + httpClient: mockClient, + outputDir: tempDir, + ); + runner.addCommand(command); + + // Run command + await runner.run(['generate-skill', videoFile.path]); + + // Just verify file creation for now + final skillDirFoo = Directory(p.join(tempDir.path, 'foo')); + expect(skillDirFoo.existsSync(), isTrue); + + final skillFile = File(p.join(skillDirFoo.path, 'SKILL.md')); + expect(skillFile.existsSync(), isTrue); + + // Verify source header was sent to Gemini + expect(geminiRequests, isNotEmpty); + expect( + geminiRequests.first, + contains('--- Raw content from https://example.com/foo.html ---'), + ); + }); + + test('logs progress and summary', () async { + videoFile = File(p.join(tempDir.path, 'dart_dev.json')); + final inputData = [ + { + 'name': 'success', + 'description': 'Desc', + 'resources': ['https://example.com/success'], + }, + { + 'name': 'fail', + 'description': 'Desc', + 'resources': ['https://example.com/fail_404'], + }, + ]; + videoFile.writeAsStringSync(jsonEncode(inputData)); + + final logs = []; + final sub = Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + + addTearDown(sub.cancel); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/success') { + return http.Response('Content', 200); + } + if (url == 'https://example.com/fail_404') { + return http.Response('Not Found', 404); + } + // Mock Gemini + if (url.contains('generativelanguage')) { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Generated Content'}, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Error', 500); + }); + + final command = GenerateSkillCommand( + httpClient: mockClient, + outputDir: tempDir, + ); + runner.addCommand(command); + + await runner.run(['generate-skill', videoFile.path]); + + // Verify Logs + expect(logs, contains(contains('Generating skill: success...'))); + expect( + logs, + contains(contains('Fetching https://example.com/success...')), + ); + expect(logs, contains(contains('Generating skill: fail...'))); + }); + + test('accepts thinking-budget option', () async { + // Setup for this test requires a configFile and skillsDir, + // which are not defined in the provided context. + // Assuming these would be defined in a real scenario or + // this test is incomplete without them. + // For now, I'll use tempDir for skillsDir and create a dummy configFile. + + final skillsDir = Directory(p.join(tempDir.path, 'skills_output')); + await skillsDir.create(); + + final configFile = File(p.join(tempDir.path, 'config.json')); + final inputData = [ + { + 'name': 'budget_test', + 'description': 'Budget test description', + 'resources': ['https://example.com/budget'], + }, + ]; + configFile.writeAsStringSync(jsonEncode(inputData)); + + final logs = []; + final sub = Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + addTearDown(sub.cancel); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/budget') { + return http.Response('Budget Content', 200); + } + if (url.contains('generativelanguage')) { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Generated Budget Content'}, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Error', 500); + }); + + final command = GenerateSkillCommand( + httpClient: mockClient, + outputDir: skillsDir, + ); + runner.addCommand(command); + + await runner.run([ + 'generate-skill', + configFile.path, + '--directory', + skillsDir.path, + '--thinking-budget', + '4000', + ]); + + expect( + logs, + contains( + contains( + 'Model: models/gemini-3.1-pro-preview, Max Output Tokens: 8192, Thinking Budget: 4000', + ), + ), + ); + final skillDirBudget = Directory(p.join(skillsDir.path, 'budget_test')); + expect(skillDirBudget.existsSync(), isTrue); + final skillFile = File(p.join(skillDirBudget.path, 'SKILL.md')); + expect(skillFile.existsSync(), isTrue); + expect( + skillFile.readAsStringSync(), + contains('Generated Budget Content\n'), + ); + }); + + test('handles invalid thinking-budget option gracefully', () async { + final skillsDir = await Directory.systemTemp.createTemp('skills_output'); + addTearDown(() => skillsDir.delete(recursive: true)); + + final configFile = File(p.join(tempDir.path, 'config.json')); + final inputData = [ + { + 'name': 'budget_test', + 'description': 'Budget test description', + 'resources': ['https://example.com/budget'], + }, + ]; + configFile.writeAsStringSync(jsonEncode(inputData)); + + final logs = []; + final sub = Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + addTearDown(sub.cancel); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/budget') { + return http.Response('Budget Content', 200); + } + if (url.contains('generativelanguage')) { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Generated Budget Content'}, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Error', 500); + }); + + final command = GenerateSkillCommand( + httpClient: mockClient, + outputDir: skillsDir, + ); + runner.addCommand(command); + + await runner.run([ + 'generate-skill', + configFile.path, + '--directory', + skillsDir.path, + '--thinking-budget', + 'invalid', + ]); + + expect( + logs, + contains(contains('Invalid thinking-budget: invalid. Skipping.')), + ); + final skillDirBudget = Directory(p.join(skillsDir.path, 'budget_test')); + expect(skillDirBudget.existsSync(), isFalse); + }); + }); +} diff --git a/tool/test/markdown_converter_test.dart b/tool/test/markdown_converter_test.dart new file mode 100644 index 00000000..6aa722ee --- /dev/null +++ b/tool/test/markdown_converter_test.dart @@ -0,0 +1,103 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:skills/src/services/markdown_converter.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarkdownConverter', () { + late MarkdownConverter converter; + + setUp(() { + converter = MarkdownConverter(); + }); + + test('converts basic HTML to Markdown', () { + const html = '

Title

Paragraph

'; + final markdown = converter.convert(html); + expect(markdown, contains('# Title')); + expect(markdown, contains('Paragraph')); + }); + + test('converts images', () { + const html = 'Alt Text'; + final markdown = converter.convert(html); + expect(markdown, equals('![Alt Text](image.png)')); + }); + + test('converts images without alt text', () { + const html = ''; + final markdown = converter.convert(html); + expect(markdown, equals('![](image.png)')); + }); + + test('converts unordered lists', () { + const html = '
  • Item 1
  • Item 2
'; + final markdown = converter.convert(html); + expect(markdown, contains('- Item 1')); + expect(markdown, contains('- Item 2')); + }); + + test('converts nested elements', () { + const html = '

Paragraph Bold

'; + final markdown = converter.convert(html); + expect(markdown, contains('Paragraph **Bold**')); + }); + + test('converts blockquotes', () { + const html = '
Quote
'; + final markdown = converter.convert(html); + expect(markdown, contains('> Quote')); + }); + + test('converts horizontal rules', () { + const html = '
'; + final markdown = converter.convert(html); + expect(markdown, contains('---')); + }); + + test('converts strikethrough', () { + const html = 'Deleted Struck Strike'; + final markdown = converter.convert(html); + expect(markdown, contains('~~Deleted~~')); + expect(markdown, contains('~~Struck~~')); + expect(markdown, contains('~~Strike~~')); + }); + + test('converts headers', () { + const html = '

H4

H5
H6
'; + final markdown = converter.convert(html); + expect(markdown, contains('#### H4')); + expect(markdown, contains('##### H5')); + expect(markdown, contains('###### H6')); + }); + + test('converts video with src', () { + const html = ''; + final markdown = converter.convert(html); + expect(markdown, equals('[Video Title](video.mp4)')); + }); + + test('converts video with source child', () { + const html = + ''; + final markdown = converter.convert(html); + expect(markdown, equals('[Video Title](video.mp4)')); + }); + + test('converts video with poster', () { + const html = + ''; + final markdown = converter.convert(html); + expect(markdown, equals('[![Video Title](poster.jpg)](video.mp4)')); + }); + + test('converts iframe', () { + const html = + ''; + final markdown = converter.convert(html); + expect(markdown, equals('[Example Iframe](https://example.com)')); + }); + }); +} diff --git a/tool/test/markdown_table_test.dart b/tool/test/markdown_table_test.dart new file mode 100644 index 00000000..32f9ac62 --- /dev/null +++ b/tool/test/markdown_table_test.dart @@ -0,0 +1,87 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:skills/src/services/markdown_converter.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarkdownConverter Tables', () { + late MarkdownConverter converter; + + setUp(() { + converter = MarkdownConverter(); + }); + + test('converts simple table', () { + const html = ''' + + + + + + + + + + + + + +
Header 1Header 2
Cell 1Cell 2
+'''; + final markdown = converter.convert(html); + expect(markdown, contains('| Header 1 | Header 2 |')); + expect(markdown, contains('|---|---|')); + expect(markdown, contains('| Cell 1 | Cell 2 |')); + }); + + test('converts table without thead', () { + const html = ''' + + + + + +
Cell 1Cell 2
+'''; + final markdown = converter.convert(html); + // Fallback: treated as table with first row as header + expect(markdown, contains('| Cell 1 | Cell 2 |')); + expect(markdown, contains('|---|---|')); + }); + + test('converts definition lists', () { + const html = ''' +
+
Term 1
+
Definition 1
+
Term 2
+
Definition 2
+
+'''; + final markdown = converter.convert(html); + expect(markdown, contains('**Term 1**')); + expect(markdown, contains(': Definition 1')); + expect(markdown, contains('**Term 2**')); + expect(markdown, contains(': Definition 2')); + }); + + test('converts details/summary', () { + const html = ''' +
+ Summary + Details content +
+'''; + // We'll preserve HTML for details as it's often supported in markdown rendering + // OR we can just output the content. + // Preserving HTML is usually safer for details. + final markdown = converter.convert(html); + expect(markdown, contains('
')); + expect(markdown, contains('Summary')); + expect(markdown, contains('Details content')); + expect(markdown, contains('
')); + }); + }); +} diff --git a/tool/test/models/skill_params_test.dart b/tool/test/models/skill_params_test.dart new file mode 100644 index 00000000..b0293775 --- /dev/null +++ b/tool/test/models/skill_params_test.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:skills/src/models/skill_params.dart'; +import 'package:test/test.dart'; + +void main() { + group('SkillParams', () { + test('fromJson parses correctly without instructions', () { + final json = { + 'name': 'test-skill', + 'description': 'Test Description', + 'resources': ['http://example.com'], + }; + final skill = SkillParams.fromJson(json); + expect(skill.name, 'test-skill'); + expect(skill.description, 'Test Description'); + expect(skill.resources, ['http://example.com']); + expect(skill.instructions, isNull); + }); + + test('fromJson parses correctly with instructions', () { + final json = { + 'name': 'test-skill', + 'description': 'Test Description', + 'instructions': 'Do not hallucinate.', + 'resources': ['http://example.com'], + }; + final skill = SkillParams.fromJson(json); + expect(skill.name, 'test-skill'); + expect(skill.description, 'Test Description'); + expect(skill.resources, ['http://example.com']); + expect(skill.instructions, 'Do not hallucinate.'); + }); + }); +} diff --git a/tool/test/services/gemini_service_test.dart b/tool/test/services/gemini_service_test.dart new file mode 100644 index 00000000..fb320a8e --- /dev/null +++ b/tool/test/services/gemini_service_test.dart @@ -0,0 +1,368 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:logging/logging.dart'; + +import 'package:skills/src/services/gemini_service.dart'; +import 'package:test/test.dart'; + +void main() { + group('GeminiService', () { + late GeminiService service; + + setUp(() { + service = GeminiService( + apiKey: 'test-api-key', + httpClient: http.Client(), + ); + }); + + group('cleanContent', () { + test('returns null for null content', () { + expect(service.cleanContent(null), isNull); + }); + + test('removes markdown code blocks around content', () { + const content = ''' +```markdown +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + + test('removes markdown code blocks with other languages', () { + const content = ''' +```text +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + + test('ignores trailing markdown code block if no start block', () { + const content = ''' +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +``` +'''; + expect(service.cleanContent(content), expected); + }); + + test('strips possible frontmatter at start', () { + const content = ''' +--- +key: value +--- +# Title +Some content +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + + test('strips possible frontmatter after some noise', () { + const content = ''' +Here is the content: +--- +key: value +--- +# Title +Some content +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + + test('preserves internal code blocks', () { + const content = ''' +# Title +Here is some code: +```dart +void main() {} +``` +'''; + const expected = ''' +# Title +Here is some code: +```dart +void main() {} +``` +'''; + expect(service.cleanContent(content), expected); + }); + + test('ensures content ends with newline', () { + const content = 'Some content'; + const expected = 'Some content\n'; + expect(service.cleanContent(content), expected); + }); + + test('handles complex nested structure', () { + const content = ''' +```markdown +--- +key: value +--- +# Title +Content with code: +```dart +print('hello'); +``` +``` +'''; + const expected = ''' +# Title +Content with code: +```dart +print('hello'); +``` +'''; + expect(service.cleanContent(content), expected); + }); + test('removes markdown code blocks with leading whitespace', () { + const content = ''' + ```markdown +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + + test( + 'removes markdown code blocks with trailing whitespace on fence', + () { + const content = ''' +```markdown +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }, + ); + + test('removes markdown code blocks with uppercase language', () { + const content = ''' +```MARKDOWN +# Title +Some content +``` +'''; + const expected = ''' +# Title +Some content +'''; + expect(service.cleanContent(content), expected); + }); + }); + }); + + group('GeminiService URL Validation', () { + late GeminiService service; + + setUp(() { + Logger.root.level = Level.ALL; + }); + + test('retries when content contains URLs', () async { + var attempt = 0; + + final client = MockClient((request) async { + attempt++; + if (attempt == 1) { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Here is a link: https://example.com'}, + ], + }, + }, + ], + }), + 200, + ); + } else { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'No links here.'}, + ], + }, + }, + ], + }), + 200, + ); + } + }); + + service = GeminiService( + apiKey: 'key', + httpClient: client, + model: 'gemini-3-pro', + ); + + final result = await service.generateSkillContent( + 'markdown', + 'name', + 'desc', + urls: ['urls'], + ); + + expect(attempt, 2); + expect(result, contains('No links here.')); + expect(result, isNot(contains('https://example.com'))); + }); + + test('fails after retries if content always contains URLs', () async { + final client = MockClient((request) async { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Link: https://example.com'}, + ], + }, + }, + ], + }), + 200, + ); + }); + + service = GeminiService( + apiKey: 'key', + httpClient: client, + model: 'gemini-3-pro', + ); + + // Capture logs to verify retries + final logs = []; + final subscription = Logger.root.onRecord.listen( + (r) => logs.add(r.message), + ); + addTearDown(subscription.cancel); + + final result = await service.generateSkillContent( + 'markdown', + 'name', + 'desc', + urls: ['urls'], + ); + + expect(result, isNull); + expect(logs, contains(contains('Retrying Gemini generation'))); + expect(logs, contains(contains('Gemini generation failed'))); + }); + }); + + group('fetchAndConvertContent', () { + late Logger logger; + + setUp(() { + logger = Logger('test'); + }); + + test('fetches and converts content successfully on 200 OK', () async { + final client = MockClient((request) async { + if (request.url.toString() == 'https://example.com/doc1') { + return http.Response('

Doc 1

', 200); + } else if (request.url.toString() == 'https://example.com/doc2') { + return http.Response('

Doc 2 content

', 200); + } + return http.Response('Not found', 404); + }); + + final result = await fetchAndConvertContent( + ['https://example.com/doc1', 'https://example.com/doc2'], + client, + logger, + ); + + expect(result, contains('Doc 1')); + expect(result, contains('Doc 2 content')); + }); + + test( + 'throws Exception on non-200 status code to save Gemini tokens', + () async { + final client = MockClient((request) async { + return http.Response('Not found', 404); + }); + + expect( + () => fetchAndConvertContent( + ['https://example.com/missing'], + client, + logger, + ), + throwsA( + isA().having( + (e) => e.toString(), + 'message', + contains('HTTP 404'), + ), + ), + ); + }, + ); + + test('throws exception on network error to save Gemini tokens', () async { + final client = MockClient((request) async { + throw http.ClientException('Connection failed'); + }); + + expect( + () => fetchAndConvertContent( + ['https://example.com/error'], + client, + logger, + ), + throwsA(isA()), + ); + }); + }); +} diff --git a/tool/test/services/markdown_converter_test.dart b/tool/test/services/markdown_converter_test.dart new file mode 100644 index 00000000..f136dc32 --- /dev/null +++ b/tool/test/services/markdown_converter_test.dart @@ -0,0 +1,107 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:skills/src/services/markdown_converter.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarkdownConverter', () { + late MarkdownConverter converter; + + setUp(() { + converter = MarkdownConverter(); + }); + + test('converts headers', () { + expect(converter.convert('

Title

'), contains('# Title')); + expect(converter.convert('

Subtitle

'), contains('## Subtitle')); + expect(converter.convert('

Section

'), contains('### Section')); + }); + + test('converts paragraphs', () { + expect(converter.convert('

Hello World

'), contains('Hello World')); + }); + + test('converts links', () { + expect( + converter.convert('Link'), + contains('[Link](https://example.com)'), + ); + }); + + test('converts bold and italic', () { + expect(converter.convert('Bold'), contains('**Bold**')); + expect( + converter.convert('Strong'), + contains('**Strong**'), + ); + expect(converter.convert('Italic'), contains('*Italic*')); + expect( + converter.convert('Emphasized'), + contains('*Emphasized*'), + ); + }); + + test('converts code', () { + expect( + converter.convert('print("hello")'), + contains('`print("hello")`'), + ); + expect( + converter.convert('
void main() {}
'), + allOf(contains('```'), contains('void main() {}')), + ); + }); + + test('converts lists', () { + const html = ''' +
    +
  • Item 1
    • +
  • Item 2
    • +
+ '''; + final md = converter.convert(html); + expect(md, contains('- Item 1')); + expect(md, contains('- Item 2')); + + const htmlOl = ''' +
    +
  1. First
    2. +
  2. Second
    3. +
+ '''; + final mdOl = converter.convert(htmlOl); + // Currently the converter uses simplified list handling (returning - for both) + expect(mdOl, contains('- First')); + expect(mdOl, contains('- Second')); + }); + + test('converts line breaks', () { + expect(converter.convert('Line 1
Line 2'), contains('Line 1\nLine 2')); + }); + + test('converts structural elements', () { + expect( + converter.convert('
Div Content
'), + contains('Div Content'), + ); + expect( + converter.convert('
Section Content
'), + contains('Section Content'), + ); + expect( + converter.convert('
Main Content
'), + contains('Main Content'), + ); + expect( + converter.convert('
Article Content
'), + contains('Article Content'), + ); + }); + + test('handles empty body', () { + expect(converter.convert(''), isEmpty); + }); + }); +} diff --git a/tool/test/skill_assertions_test.dart b/tool/test/skill_assertions_test.dart new file mode 100644 index 00000000..c78f52d8 --- /dev/null +++ b/tool/test/skill_assertions_test.dart @@ -0,0 +1,123 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as path; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + group('Skill Assertions', () { + final skillsDir = Directory('../skills'); + + test('skills directory exists', () { + expect( + skillsDir.existsSync(), + isTrue, + reason: 'skills directory should exist', + ); + }); + + final skillDirs = skillsDir.listSync().whereType(); + + for (final dir in skillDirs) { + final skillName = path.basename(dir.path); + + group('Skill: $skillName', () { + test('has exactly one SKILL.md', () { + final files = dir + .listSync() + .whereType() + .where((f) => path.basename(f.path) == 'SKILL.md') + .toList(); + expect( + files.length, + equals(1), + reason: 'Should have exactly one SKILL.md file', + ); + }); + + test('SKILL.md has valid frontmatter', () { + final skillFile = File(path.join(dir.path, 'SKILL.md')); + if (!skillFile.existsSync()) return; + + final content = skillFile.readAsStringSync(); + final frontmatterMatch = RegExp( + r'^---\n([\s\S]*?)\n---', + ).firstMatch(content); + + expect( + frontmatterMatch, + isNotNull, + reason: + 'SKILL.md should start with YAML frontmatter bounded by ---', + ); + + final yamlContent = frontmatterMatch!.group(1)!; + final yaml = loadYaml(yamlContent) as Map; + + // 1. Name matches directory name + expect( + yaml['name'], + equals(skillName), + reason: 'name in SKILL.md should match directory name', + ); + + expect( + skillName, + matches(r'^[a-z0-9]+(?:-[a-z0-9]+)*(?:_[0-9]+)?$'), + reason: + 'Skill name should be kebab-case (lowercase, numbers, hyphens), ' + 'with an optional numeric suffix starting with underscore (e.g. _2)', + ); + + // 2. Name validation strategy + final isDocSkill = + (yaml['name'] as String).startsWith('dart-docs-') || + (yaml['name'] as String).startsWith('flutter-docs-'); + + // 3. Required fields + expect( + yaml, + contains('description'), + reason: 'Should have description', + ); + + // Metadata validation (Strict only for doc skills) + if (isDocSkill) { + expect( + yaml, + contains('metadata'), + reason: 'Doc skills should have metadata', + ); + final metadata = yaml['metadata'] as Map; + expect( + metadata, + contains('url'), + reason: 'Metadata should have url', + ); + expect( + metadata, + contains('model'), + reason: 'Metadata should have model', + ); + + final hasLastModified = metadata.containsKey('last_modified'); + expect( + hasLastModified, + isTrue, + reason: 'Metadata should have last_modified', + ); + + // URL validation + final url = Uri.tryParse(metadata['url'] as String); + expect(url, isNotNull, reason: 'URL should be valid'); + expect(url!.hasScheme, isTrue, reason: 'URL should have scheme'); + } + }); + }); + } + }); +} diff --git a/tool/test/validate_skills_test.dart b/tool/test/validate_skills_test.dart new file mode 100644 index 00000000..1486b66d --- /dev/null +++ b/tool/test/validate_skills_test.dart @@ -0,0 +1,515 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:skills/src/commands/validate_skill_command.dart'; +import 'package:test/test.dart'; + +void main() { + group('ValidateSkillCommand', () { + late CommandRunner runner; + late Directory tempDir; + late Directory skillsDir; + late Directory validationDir; + late MockClient mockClient; + final logs = []; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('validate_skills_test'); + skillsDir = Directory(p.join(tempDir.path, 'skills')); + await skillsDir.create(); + validationDir = Directory(p.join(tempDir.path, 'validation')); + await validationDir.create(); + + mockClient = MockClient((request) async { + return http.Response('', 200); + }); + + // Capture logs + Logger.root.level = Level.INFO; + Logger.root.onRecord.listen((record) { + logs.add(record.message); + }); + logs.clear(); + }); + + tearDown(() async { + await tempDir.delete(recursive: true); + logs.clear(); + }); + + test('validates single skill defined in config', () async { + const skillName = 'test-skill'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + final skillFile = File(p.join(skillDir.path, 'SKILL.md')); + await skillFile.writeAsString('content'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Description', + 'resources': ['https://example.com'], + }, + ]), + ); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect(logs, contains('Validating skill: $skillName...')); + }); + + test('validates and grades single skill', () async { + const skillName = 'test-skill'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + final skillFile = File(p.join(skillDir.path, 'SKILL.md')); + await skillFile.writeAsString('Existing content'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Description', + 'resources': ['https://example.com/source'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/source') { + return http.Response( + '

Source

', + 200, + ); + } + if (url.contains('generativelanguage')) { + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + { + 'text': + '# Validation Report\n\n' + '- Accuracy: High\n' + '- Structure: Correct\n' + '- Completeness: Good\n\n' + 'Conclusion: Valid\n' + 'Similarity Score: 85', + }, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect(logs, contains('Validating skill: $skillName...')); + expect(logs, contains(contains('Validation report written to'))); + }); + + test('validates all skills in config', () async { + const skill1Name = 'skill1'; + final skill1Dir = Directory(p.join(skillsDir.path, skill1Name)); + await skill1Dir.create(); + File(p.join(skill1Dir.path, 'SKILL.md')).writeAsStringSync('content'); + + const skill2Name = 'skill2'; + final skill2Dir = Directory(p.join(skillsDir.path, skill2Name)); + await skill2Dir.create(); + File(p.join(skill2Dir.path, 'SKILL.md')).writeAsStringSync('content'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skill1Name, + 'description': 'Desc 1', + 'resources': ['https://example.com/1'], + }, + { + 'name': skill2Name, + 'description': 'Desc 2', + 'resources': ['https://example.com/2'], + }, + ]), + ); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect(logs, contains('Validating skill: $skill1Name...')); + expect(logs, contains('Validating skill: $skill2Name...')); + }); + + test('logs severe error when config file not found', () async { + final path = p.join(tempDir.path, 'NON_EXISTENT.json'); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand(outputDir: skillsDir, httpClient: mockClient), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', path]); + }, getCurrentDirectory: () => tempDir); + + expect(logs, contains('Configuration file not found: $path')); + }); + + test('logs warning when existing skill file not found', () async { + const skillName = 'missing-skill'; + // Do NOT create skill directory or file + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Desc', + 'resources': ['https://example.com'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com') { + return http.Response('Content', 200); + } + if (url.contains('generativelanguage')) { + return http.Response( + '{"candidates": [{"content": {"parts": [{"text": "Generated Content"}]}}]}', + 200, + ); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand(outputDir: skillsDir, httpClient: mockClient), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect(logs, contains(contains('Existing skill file not found'))); + }); + + test('logs warning when failed to fetch URL', () async { + const skillName = 'fail-skill'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + File(p.join(skillDir.path, 'SKILL.md')).writeAsStringSync('content'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Desc', + 'resources': ['https://example.com/fail'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + // Throw to trigger the 'Error fetching' catch block + throw Exception('Network Error'); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand(outputDir: skillsDir, httpClient: mockClient), + ); + + await runner.run(['validate-skill', configFile.path]); + + expect( + logs, + contains( + contains('Error validating $skillName: Exception: Network Error'), + ), + ); + }); + + test( + 'logs severe error when grading fails repeatedly/exceptionally', + () async { + const skillName = 'grade-fail'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + File(p.join(skillDir.path, 'SKILL.md')).writeAsStringSync('content'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Desc', + 'resources': ['https://example.com/source'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + if (request.url.toString() == 'https://example.com/source') { + return http.Response('# Source', 200); + } + if (request.url.toString().contains('generativelanguage')) { + // Validation (grading) fails + throw Exception('Gemini API Error'); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await runner.run(['validate-skill', configFile.path]); + + expect( + logs, + contains(contains('Failed to generate validation report')), + ); + }, + ); + + test('logs severe error when Gemini grading throws exception', () async { + const skillName = 'gemini-fail'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + // Ensure name matches to avoid name mismatch error + File( + p.join(skillDir.path, 'SKILL.md'), + ).writeAsStringSync('name: $skillName\nContent'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Desc', + 'resources': ['https://example.com/source'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + if (request.url.toString() == 'https://example.com/source') { + return http.Response('# Source', 200); + } + if (request.url.toString().contains('generativelanguage')) { + // Validation (grading) throws + throw Exception('Gemini API Error'); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await runner.run(['validate-skill', configFile.path]); + + expect(logs, contains(contains('Failed to generate validation report'))); + }); + + test('logs severe error when skill name/description mismatch', () async { + const skillName = 'mismatch-skill'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + // Write content WITHOUT name/description + final skillFile = File(p.join(skillDir.path, 'SKILL.md')); + await skillFile.writeAsString('Invalid content without frontmatter'); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Expected Description', + 'resources': ['https://example.com/source'], + }, + ]), + ); + + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/source') { + return http.Response('# Source', 200); + } + if (url.contains('generativelanguage')) { + return http.Response( + '{"candidates": [{"content": {"parts": [{"text": "Generated Content"}]}}]}', + 200, + ); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + validationDir: validationDir, + httpClient: mockClient, + ), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect( + logs, + contains(contains('Validation Failed: Skill name mismatch')), + ); + expect( + logs, + contains(contains('Validation Failed: Skill name mismatch')), + ); + // Description check was removed from the command + expect( + logs, + isNot( + contains(contains('Validation Failed: Skill description mismatch')), + ), + ); + }); + + test('sends source URL header to Gemini during regeneration', () async { + const skillName = 'header-skill'; + final skillDir = Directory(p.join(skillsDir.path, skillName)); + await skillDir.create(); + final skillFile = File(p.join(skillDir.path, 'SKILL.md')); + await skillFile.writeAsString(''' +name: $skillName +description: Desc +--- +Content +'''); + + final configFile = File(p.join(tempDir.path, 'config.json')); + await configFile.writeAsString( + jsonEncode([ + { + 'name': skillName, + 'description': 'Desc', + 'resources': ['https://example.com/source'], + }, + ]), + ); + + final geminiRequests = []; + final mockClient = MockClient((request) async { + final url = request.url.toString(); + if (url == 'https://example.com/source') { + return http.Response('Source Content', 200); + } + if (url.contains('generativelanguage')) { + geminiRequests.add(request.body); + return http.Response( + jsonEncode({ + 'candidates': [ + { + 'content': { + 'parts': [ + {'text': 'Generated Content\nSimilarity Score: 50'}, + ], + }, + }, + ], + }), + 200, + ); + } + return http.Response('Not Found', 404); + }); + + runner = CommandRunner('skills', 'Test runner') + ..addCommand( + ValidateSkillCommand( + outputDir: skillsDir, + httpClient: mockClient, + validationDir: validationDir, + ), + ); + + await IOOverrides.runZoned(() async { + await runner.run(['validate-skill', configFile.path]); + }, getCurrentDirectory: () => tempDir); + + expect(geminiRequests, isNotEmpty); + expect( + geminiRequests.first, + contains('--- Raw content from https://example.com/source ---'), + ); + }); + }); +} diff --git a/tool/test/yaml_assets_test.dart b/tool/test/yaml_assets_test.dart new file mode 100644 index 00000000..0611cde2 --- /dev/null +++ b/tool/test/yaml_assets_test.dart @@ -0,0 +1,184 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + final resourcesDir = Directory( + p.join(Directory.current.path, '..', 'tool', 'resources'), + ); + + test('tool/resources directory exists', () { + expect( + resourcesDir.existsSync(), + isTrue, + reason: 'tool/resources directory should exist', + ); + }); + + if (!resourcesDir.existsSync()) return; + + final yamlFiles = resourcesDir + .listSync() + .where((e) => e.path.endsWith('.yaml')) + .cast(); + + test('tool/resources contains YAML files', () { + expect( + yamlFiles, + isNotEmpty, + reason: 'No YAML files found in tool/resources', + ); + }); + + final client = http.Client(); + // We can't use addTearDown here because it must be within a test or setUp/tearDown. + // We'll use tearDownAll for the group cleanup. + tearDownAll(client.close); + + for (final file in yamlFiles) { + group(p.basename(file.path), () { + final content = file.readAsStringSync(); + YamlList? yaml; + try { + yaml = loadYaml(content) as YamlList; + } catch (e) { + test('is valid YAML list', () { + fail('Failed to decode YAML: $e'); + }); + return; + } + + test('structure is a List', () { + expect(yaml, isA(), reason: 'Root must be a list'); + expect(yaml, isNotEmpty, reason: 'Root list must not be empty'); + }); + + // yaml is known to be non-null here because of the try/catch above and type cast + for (var i = 0; i < yaml.length; i++) { + final item = yaml[i]; + if (item is! YamlMap && item is! Map) { + test('Item $i is a Map', () { + fail('Item $i is not a Map'); + }); + continue; + } + + final name = item['name'] as String? ?? 'Item $i'; + test('name is kabob-case', () { + expect( + name, + matches(RegExp(r'^[a-z0-9]+(-[a-z0-9]+)*$')), + reason: + 'Skill name must be kabob-case (e.g. abc-def). See https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices#naming-conventions', + ); + }); + + if (p.basename(file.path) == 'flutter_skills.yaml') { + test('name starting with "flutter-"', () { + expect( + name, + startsWith('flutter-'), + reason: + 'All skills in flutter_skills.yaml must start with "flutter-"', + ); + }); + } else if (p.basename(file.path) == 'dart_skills.yaml') { + test('name starting with "dart-"', () { + expect( + name, + startsWith('dart-'), + reason: 'All skills in dart_skills.yaml must start with "dart-"', + ); + }); + } + + group(name, () { + test('has required fields', () { + expect(item, contains('name'), reason: 'Missing "name"'); + expect( + item, + contains('description'), + reason: 'Missing "description"', + ); + expect(item, contains('resources'), reason: 'Missing "resources"'); + }); + + test('resources is a non-empty List', () { + expect( + item['resources'], + isA(), + reason: '"resources" must be a list', + ); + final resources = item['resources'] as YamlList; + expect( + resources, + isNotEmpty, + reason: '"resources" must not be empty', + ); + }); + + if (item['resources'] is YamlList) { + final resources = item['resources'] as YamlList; + for (final resource in resources) { + final url = resource as String; + if (p.basename(file.path) == 'flutter_skills.yaml') { + // Validate that all Flutter skill resources come from the official Flutter documentation. + test('URL starts with https://docs.flutter.dev/', () { + expect( + url, + startsWith('https://docs.flutter.dev/'), + reason: + 'All resources in flutter_skills.yaml must be from docs.flutter.dev', + ); + }); + } else if (p.basename(file.path) == 'dart_skills.yaml') { + // Validate that all Dart skill resources come from the official Dart documentation. + test('URL starts with https://dart.dev/', () { + expect( + url, + startsWith('https://dart.dev/'), + reason: + 'All resources in dart_skills.yaml must be from dart.dev', + ); + }); + } + test('URL: $url', () => _validateResource(client, url)); + } + } + }); + } + }); + } +} + +Future _validateResource(http.Client client, String url) async { + expect(url, startsWith('https://'), reason: 'URL must start with https://'); + + try { + final response = await client.head(Uri.parse(url)); + if (response.statusCode == 405) { + // Fallback to GET if HEAD is not allowed + final getResponse = await client.get(Uri.parse(url)); + expect( + getResponse.statusCode, + equals(200), + reason: 'GET returned ${getResponse.statusCode}', + ); + } else { + expect( + response.statusCode, + equals(200), + reason: 'HEAD returned ${response.statusCode}', + ); + } + } catch (e) { + fail('Failed to connect: $e'); + } +}