Separate weekly openApi-doc refresh and command-metadata refresh

.azure-pipelines/command-metadata-refresh.yml

@@ -0,0 +1,150 @@
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# This pipeline will be extended to the OneESPT template
+# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage
+# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section.
+# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section. 
+
+name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
+parameters:
+- name: BuildAgent
+  displayName: Build Agent
+  default: 1es-windows-ps-compute-m
+- name: BaseBranch
+  displayName: Base Branch
+  default: main
+- name: Test
+  type: boolean
+  default: true
+- name: Pack
+  type: boolean
+  default: false
+- name: Sign
+  type: boolean
+  default: false
+- name: BumpModuleVersion
+  type: boolean
+  default: false
+- name: CreatePullRequest
+  type: boolean
+  default: true
+
+variables:
+  BuildAgent: ${{ parameters.BuildAgent }}
+  Branch: "ModuleCommandMetadataRefresh"
+  BaseBranch: ${{ parameters.BaseBranch }}
+
+trigger:
+  branches:
+    include:
+    - main
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool: $(BuildAgent)
+    sdl:
+      binskim:
+        enabled: false
+        justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds."
+      credscan:
+        suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json
+      policheck:
+        exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml
+    customBuildTags:
+    - ES365AIMigrationTooling
+    stages:
+    - stage: stage
+      jobs:
+      - job: CreateMetadataRefreshBranch
+        displayName: Create Metadata Refresh Branch
+        timeoutInMinutes: 1200
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: ${{ parameters.BaseBranch }}
+        - task: PowerShell@2
+          name: "ComputeBranch"
+          displayName: "Compute Module Command Metadata Refresh Branch Name"
+          inputs:
+            targetType: inline
+            script: |
+              $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
+              Write-Host "##vso[task.setvariable variable=RefreshBranch;isOutput=true]$branch" 
+        - task: Bash@3
+          displayName: "Create Module Command Metadata Refresh Branch"
+          inputs:
+            targetType: inline
+            script: |
+              git status
+              git branch $(ComputeBranch.RefreshBranch)
+              git checkout $(ComputeBranch.RefreshBranch)
+              git status               
+
+      - job: MsGraphPsSdkModuleCommandMetadataGeneration
+        dependsOn: CreateMetadataRefreshBranch
+        displayName: Microsoft Graph PowerShell SDK Generation
+        condition: succeeded()
+        timeoutInMinutes: 840
+        variables:
+          RefreshBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.RefreshBranch'] ]
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: $(RefreshBranch)
+        - template: .azure-pipelines/common-templates/install-tools.yml@self
+        - template: .azure-pipelines/common-templates/security-pre-checks.yml@self
+        - template: .azure-pipelines/generation-templates/authentication-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/workload-modules.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/meta-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
+        - template: .azure-pipelines/common-templates/security-post-checks.yml@self
+
+        - ${{ if eq(parameters.BumpModuleVersion, true) }}:  
+          - task: PowerShell@2
+            name: CalculateAndBumpModuleVersion
+            displayName: Calculate and bump module version
+            condition: succeeded()
+            inputs:
+              pwsh: true
+              targetType: inline
+              script: |
+                . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
+          - task: Bash@3
+            displayName: Push version bump changes
+            env:
+              GITHUB_TOKEN: $(GITHUB_TOKEN)
+            inputs:
+              targetType: inline
+              script: |
+                git status
+                git add "$(System.DefaultWorkingDirectory)/config/ModuleMetadata.json"
+                git commit -m 'Bump module versions after metadata generation.'
+                git push "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git"
+                git status
+
+        - ${{ if eq(parameters.CreatePullRequest, true) }}:
+          - template: .azure-pipelines/common-templates/create-pr.yml@self
+            parameters:
+              BaseBranch: $(BaseBranch)
+              TargetBranch: $(RefreshBranch)
+              Title: "[v2] Module Command Metadata Refresh"
+              Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

.azure-pipelines/common-templates/create-pr.yml

@@ -4,7 +4,7 @@
 parameters:
   - name: BaseBranch
     type: string
-    default: dev
+    default: main
   - name: TargetBranch
     type: string
     default: ""

.azure-pipelines/common-templates/download-openapi-docs.yml

@@ -73,25 +73,19 @@ steps:
           $diff = git diff --name-only
           $ModulesWithChanges = @{}
           $diff | %{
-            if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2))
-            {
-              $ModulesWithChanges.Add($matches.2, $matches.1)
+            if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') {
+              $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' }
+              $moduleName = "$($matches[2])_$version"
+              if (!$ModulesWithChanges.ContainsKey($moduleName)) {
+                $ModulesWithChanges.Add($moduleName, $matches[1])
+              }
             }
           }
           $ModuleNames = $ModulesWithChanges.Keys
+          Write-Host "Modules with changes: $ModuleNames"
         }
         Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames"
 
-  - task: PowerShell@2
-    name: CalculateAndBumpModuleVersion
-    displayName: Calculate and bump module version
-    condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''))
-    inputs:
-      pwsh: true
-      targetType: inline
-      script: |
-        . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
-
   - task: PowerShell@2
     name: BuildOpenApiMetadataDetectionTool
     displayName: Build tool for detecting metadata changes
@@ -128,4 +122,4 @@ steps:
 # References
 # [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables
 # [1] https://hub.github.com/hub-pull-request.1.html
-# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token
+# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token

.azure-pipelines/generation-templates/individual-workload-module.yml

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+
+parameters:
+  - name: Test
+    type: boolean
+    default: true
+  - name: ModuleName
+    type: string
+    default: ""    
+  - name: ModuleVersion
+    type: string
+    default: ""
+
+steps:
+  - task: PowerShell@2
+    displayName: Generate Workload Modules
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+  - template: ../common-templates/guardian-analyzer.yml
+
+  - task: PowerShell@2
+    displayName: Test Workload Modules
+    enabled: false
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+
+  - task: PowerShell@2
+    displayName: Find Duplicate Commands
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/"
+