From c3f9d5bc7f4147265cb3665c6c9d0906dad38714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peer=20Gr=C3=B8nnerup?= Date: Wed, 11 Feb 2026 15:47:55 +0100 Subject: [PATCH] Added support for Warehouse item and force switch on cli get commands --- README.md | 8 +++++ .../scripts/fabric_feature_maintainance.py | 8 ++--- automation/scripts/fabric_gitsync_env.py | 2 +- automation/scripts/fabric_release.py | 4 +-- automation/scripts/fabric_setup.py | 28 ++++++++------- .../scripts/generate_connection_string.py | 30 ++++++++-------- ...cale_bind_semantic_model_connection_dev.py | 2 +- .../scripts/modules/fabric_cli_functions.py | 36 ++++++++++++++++--- .../scripts/utils_build_parameter_file.py | 10 +++--- .../utils_build_parameter_file_dynamic.py | 2 +- 10 files changed, 84 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index f0e5e38..3b69880 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,14 @@ To use FabricOps in your environment, you'll need: - Appropriate Fabric workspace permissions - Project Administrator role (Azure DevOps) or equivalent GitHub permissions +### Fabric CLI Version Requirement +**Fabric CLI version 1.0.0+ is required** + +Version 1.0.0 introduced the `-f`/`--force` flag in the `get` command to suppress warnings on sensitivity labels. The release notes state: +> "Added a confirmation prompt in get to acknowledge that exported items do not include sensitivity labels; use -f to skip." + +Versions <1.0.0 do not support the `-f` switch and will cause automation scripts to fail. + ## Project Structure ``` diff --git a/automation/scripts/fabric_feature_maintainance.py b/automation/scripts/fabric_feature_maintainance.py index c8df4a0..a88e544 100644 --- a/automation/scripts/fabric_feature_maintainance.py +++ b/automation/scripts/fabric_feature_maintainance.py @@ -69,7 +69,7 @@ def filter_layers_by_branch(layers, branch_name_trimmed): if fabcli.run_command(f"exists {workspace_name_escaped}.Workspace").replace("*", "").strip().lower() == "false": misc.print_info(f"Creating workspace '{workspace_name}'...", bold=True, end="") fabcli.run_command(f"create '{workspace_name_escaped}.Workspace' -P capacityname={capacity_name}") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() misc.print_success(" ✔", bold=True) if permissions: @@ -105,7 +105,7 @@ def filter_layers_by_branch(layers, branch_name_trimmed): connection_name = git_settings.get("myGitCredentials").get("connection_name").format(identity_id=identity_id, identity_username=identity_username) if fabcli.connection_exists(connection_name): - connection_id = fabcli.run_command(f"get .connections/{connection_name}.Connection -q id") + connection_id = fabcli.run_command(f"get .connections/{connection_name}.Connection -q id -f") git_settings["myGitCredentials"].pop("connection_name", None) # Remove connection name git_settings["myGitCredentials"]["connectionId"] = connection_id # Add connection id required by Fabric REST API @@ -135,7 +135,7 @@ def filter_layers_by_branch(layers, branch_name_trimmed): misc.print_info(f"{workspace_name} already exist. Feature workspace creation skipped!", bold=True) if layer_definition.get("git_synchronize_on_commit", False) and not layer_definition.get("git_disconnect_after_initialize", False): misc.print_info(f" • Synchronizing workspace {workspace_name_escaped} with latest changes from Git...", end="") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() git_status = fabcli.get_git_status(workspace_id) @@ -158,7 +158,7 @@ def filter_layers_by_branch(layers, branch_name_trimmed): workspace_name_escaped = workspace_name.replace("/", "\\/") if layer_definition.get("git_synchronize_on_commit", False) and not layer_definition.get("git_disconnect_after_initialize", False): misc.print_info(f"Synchronizing workspace {workspace_name_escaped} with latest changes from Git repo...", bold=True, end="") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() git_status = fabcli.get_git_status(workspace_id) if git_status and git_status.get("workspaceHead") == git_status.get("remoteCommitHash"): diff --git a/automation/scripts/fabric_gitsync_env.py b/automation/scripts/fabric_gitsync_env.py index f09f233..77f4c28 100644 --- a/automation/scripts/fabric_gitsync_env.py +++ b/automation/scripts/fabric_gitsync_env.py @@ -37,7 +37,7 @@ if layer_definition.get("git_synchronize_on_commit", True) and not layer_definition.get("git_disconnect_after_initialize", False): misc.print_info(f"Synchronizing workspace {workspace_name_escaped} with latest changes from Git repo...", bold=True, end="") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() git_status = fabcli.get_git_status(workspace_id) if git_status is None: diff --git a/automation/scripts/fabric_release.py b/automation/scripts/fabric_release.py index 4b878d8..a2747fe 100644 --- a/automation/scripts/fabric_release.py +++ b/automation/scripts/fabric_release.py @@ -71,7 +71,7 @@ workspace_name = solution_name.format(layer=layer, environment=environment) workspace_name_escaped = workspace_name.replace("/", "\\/") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() misc.print_subheader(f"Running release to workspace {workspace_name}!") @@ -136,7 +136,7 @@ # Now bind all semantic models to this lakehouse for semantic_model_name in semantic_models: - semantic_model_id = fabcli.run_command(f"get '/{workspace_name}.Workspace/{semantic_model_name}.SemanticModel' -q id").strip() + semantic_model_id = fabcli.run_command(f"get '/{workspace_name}.Workspace/{semantic_model_name}.SemanticModel' -q id -f").strip() if not semantic_model_id: misc.print_warning(f"Semantic model '{semantic_model_name}' not found in workspace {workspace_name}. Skip binding.") continue diff --git a/automation/scripts/fabric_setup.py b/automation/scripts/fabric_setup.py index 37377fe..c1f8611 100644 --- a/automation/scripts/fabric_setup.py +++ b/automation/scripts/fabric_setup.py @@ -33,6 +33,7 @@ # Authenticate fabcli.run_command("config set encryption_fallback_enabled true") +fabcli.run_command("config set folder_listing_enabled true") fabcli.run_command(f"auth login -u {client_id} -p {client_secret} --tenant {tenant_id}") # Load JSON environment files (main and environment specific) and merge @@ -145,7 +146,7 @@ else: misc.print_warning(f" ⚠ Already exists", bold=True) - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() # Update layer_definition layer_definition["workspace_id"] = workspace_id @@ -180,7 +181,7 @@ print_item_header = True for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: has_item_connections = True if not item.get("skip_item_creation", False): @@ -188,11 +189,12 @@ print(f" • Creating workspace items:") print_item_header = False - misc.print_info(f" ◦ {item_type}: {item.get("item_name")}...", end="") + item_folder = f'{item.get("item_folder")}/' if item.get("item_folder") else "" + misc.print_info(f" ◦ {item_type}: {item_folder}{item.get("item_name")}...", end="") - if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'): - fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'") - item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}", retry_count=2) + if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'): + fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'") + item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}", retry_count=2) if item_type in {"Lakehouse"}: # Wait until SQL endpoint provisioning completes; treat missing metadata as still provisioning @@ -209,7 +211,7 @@ break print(".", end="") time.sleep(2) - item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") + item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}") if item["item_metadata"]: misc.print_success(" ✔") @@ -269,7 +271,7 @@ for workspace_name, workspace_identity in workspace_identity_acl.items(): misc.print_info(f" • Assigning workspace identity {workspace_identity} to {workspace_name}...", end="") try: - identity_id = fabcli.run_command(f"get {workspace_identity}.Workspace -q workspaceIdentity.servicePrincipalId").strip() + identity_id = fabcli.run_command(f"get {workspace_identity}.Workspace -q workspaceIdentity.servicePrincipalId -f").strip() fabcli.run_command(f"acl set {workspace_name}.Workspace -I {identity_id} -R admin -f") misc.print_success(" ✔") except Exception as e: @@ -290,7 +292,7 @@ for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer, environment=environment) item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") #print(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") @@ -298,12 +300,12 @@ if item["item_metadata"]: server = ( - item.get("item_metadata").get("properties").get("sqlEndpointProperties").get("connectionString") if item_type == "Lakehouse" else - item.get("item_metadata").get("properties").get("serverFqdn") + item.get("item_metadata").get("properties").get("serverFqdn") if item_type == "SQLDatabase" else + item.get("item_metadata").get("properties").get("connectionString") ) database = ( - item.get("item_name") if item_type == "Lakehouse" else + item.get("item_name") if item_type in ("Lakehouse","Warehouse") else item.get("item_metadata").get("properties").get("databaseName") ) @@ -383,7 +385,7 @@ if layer_definition.get("items"): for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer, environment=environment) misc.print_info(f" • Deleting connection '{connection_name}'... ", bold=False, end="") if fabcli.run_command(f"exists .connections/{connection_name}.Connection").replace("*", "").strip().lower() == "true": diff --git a/automation/scripts/generate_connection_string.py b/automation/scripts/generate_connection_string.py index 6b90eb2..3dc8be1 100644 --- a/automation/scripts/generate_connection_string.py +++ b/automation/scripts/generate_connection_string.py @@ -35,21 +35,21 @@ solution_name = env_definition.get("name") workspace_name = solution_name.format(layer=layer, environment=environment) -workspace_name_escaped = workspace_name.replace("/", "\\/") -sqldb_item = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{database}.SQLDatabase") - -# Example: You may want to adjust the server/database names per environment -server = sqldb_item.get('properties').get('serverFqdn') -database = sqldb_item.get("properties").get("databaseName") - -connection_string = ( - f"Server={server};" - f"Database={database};" - f"Authentication=Active Directory Service Principal;" - f"User Id={client_id};" - f"Password={client_secret};" - f"Encrypt=True;" - f"Connection Timeout=60;" +item_type = None +for env_item_type in env_definition.get("layers").get(layer).get("items"): + if env_item_type in ["Warehouse","SQLDatabase"]: + for item in env_definition.get("layers").get(layer).get("items").get(env_item_type): + item_name = item.get("item_name") + if item_name == database: + item_type = env_item_type + break + +connection_string = fabcli.generate_connection_string( + workspace_name=workspace_name, + item_type=item_type, + database=database, + client_id=client_id, + client_secret=client_secret ) with open(args.output_file, "w") as f: diff --git a/automation/scripts/locale/locale_bind_semantic_model_connection_dev.py b/automation/scripts/locale/locale_bind_semantic_model_connection_dev.py index c58e0b0..e992899 100644 --- a/automation/scripts/locale/locale_bind_semantic_model_connection_dev.py +++ b/automation/scripts/locale/locale_bind_semantic_model_connection_dev.py @@ -40,7 +40,7 @@ # Resolve lakehouse connection and SQL endpoint information workspace_name = solution_name.format(layer=model_layer, environment=dev_environment).replace("/", "\\/") - workspace_id = fabcli.run_command(f"get '{workspace_name}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name}.Workspace' -q id -f").strip() semantic_model_id = fabcli.get_item_id(f"/{workspace_name}.Workspace/{semantic_model_name}.SemanticModel", retry_count=2) diff --git a/automation/scripts/modules/fabric_cli_functions.py b/automation/scripts/modules/fabric_cli_functions.py index 079d8dc..a819158 100644 --- a/automation/scripts/modules/fabric_cli_functions.py +++ b/automation/scripts/modules/fabric_cli_functions.py @@ -34,7 +34,7 @@ def run_command(command: str) -> str: def get_item(item_path: str, retry_count: int = 0): for attempt in range(retry_count + 1): try: - cli_response = run_command(f"get {item_path} -q .") + cli_response = run_command(f"get {item_path} -q . -f") return json.loads(cli_response) except Exception as e: if attempt < retry_count: @@ -46,7 +46,7 @@ def get_item(item_path: str, retry_count: int = 0): def get_item_id(item_path: str, retry_count: int = 0): for attempt in range(retry_count + 1): try: - cli_response = run_command(f"get {item_path} -q id") + cli_response = run_command(f"get {item_path} -q id -f") return cli_response.strip() except Exception as e: if attempt < retry_count: @@ -61,7 +61,7 @@ def get_connection(connection_identifier): response = run_command(f"api -X get {connection_url} ") return json.loads(response) else: - return json.loads(run_command(f"get .connections/{connection_identifier}.Connection -q .")) + return json.loads(run_command(f"get .connections/{connection_identifier}.Connection -q . -f")) def connection_exists(connection_identifier): @@ -207,6 +207,8 @@ def create_fabric_connection(connection_name, connection_type, credential_type, return None + + def add_connection_roleassignment(connection_id, identity_id, identity_type, role): body = { "principal": { @@ -303,4 +305,30 @@ def poll_operation_status(operation_id): def takeover_semantic_model(workspace_id, semantic_model_id): takeover_url = f"groups/{workspace_id}/datasets/{semantic_model_id}/Default.TakeOver" response = run_command(f"api -A powerbi -X post {takeover_url}") - return json.loads(response) \ No newline at end of file + return json.loads(response) + + +def generate_connection_string(workspace_name, item_type, database, client_id, client_secret): + print(f"Generating connection string for {item_type} '{database}' in workspace '{workspace_name}'...") + workspace_name_escaped = workspace_name.replace("/", "\\/") + sqldb_item = get_item(f"/{workspace_name_escaped}.Workspace/{database}.{item_type}") + print(sqldb_item) + if item_type == "SQLDatabase": + server = sqldb_item.get('properties').get('serverFqdn') + database = sqldb_item.get("properties").get("databaseName") + elif item_type == "Lakehouse": + server = sqldb_item.get("properties").get('sqlEndpointProperties').get('connectionString') + else: + server = sqldb_item.get('properties').get('connectionString') + + connection_string = ( + f"Server={server};" + f"Database={database};" + f"Authentication=Active Directory Service Principal;" + f"User Id={client_id};" + f"Password={client_secret};" + f"Encrypt=True;" + f"Connection Timeout=60;" + ) + + return connection_string \ No newline at end of file diff --git a/automation/scripts/utils_build_parameter_file.py b/automation/scripts/utils_build_parameter_file.py index da90de4..6cc1f33 100644 --- a/automation/scripts/utils_build_parameter_file.py +++ b/automation/scripts/utils_build_parameter_file.py @@ -56,7 +56,7 @@ for layer_name, layer_definition in layers.items(): workspace_name = solution_name.format(layer=layer_name, environment=environment) workspace_name_escaped = workspace_name.replace("/", "\\/") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() print(f"Getting data for {workspace_id}, {workspace_name}") if(misc.is_guid(workspace_id)): workspace_items = fabcli.list_all_workspace_items(workspace_id) @@ -77,11 +77,11 @@ "type": item.get("type") } - if item.get("type") in {"Lakehouse", "SQLDatabase"}: + if item.get("type") in {"Lakehouse", "SQLDatabase", "Warehouse"}: item_details = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('displayName')}.{item.get('type')}", retry_count=2) fabric_item.update({ - "connectionString": item_details.get("properties").get("connectionString") if item.get("type") == "SQLDatabase" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") , - "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else None, + "connectionString": item_details.get("properties").get("connectionString") if item.get("type") != "Lakehouse" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") , + "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else item_details.get("displayName") if item.get("type") == "Warehouse" else None, "serverFqdn": item_details.get("properties").get("serverFqdn") if item.get("type") == "SQLDatabase" else None, "sqlEndpointId": item_details.get("properties").get("sqlEndpointProperties").get("id") if item.get("type") == "Lakehouse" else None, }) @@ -92,7 +92,7 @@ if layer_definition.get("items"): for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer_name, environment=environment) if fabcli.connection_exists(connection_name): connection = fabcli.get_item(f".connections/{connection_name}.Connection") diff --git a/automation/scripts/utils_build_parameter_file_dynamic.py b/automation/scripts/utils_build_parameter_file_dynamic.py index e3d68ba..4550f22 100644 --- a/automation/scripts/utils_build_parameter_file_dynamic.py +++ b/automation/scripts/utils_build_parameter_file_dynamic.py @@ -60,7 +60,7 @@ workspace_name_escaped = workspace_name.replace("/", "\\/") misc.print_info(f" Scanning workspace: {workspace_name}...", bold=False, end="") - workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id").strip() + workspace_id = fabcli.run_command(f"get '{workspace_name_escaped}.Workspace' -q id -f").strip() if misc.is_guid(workspace_id): print(" ✔")