Re-factor the build script and add auto update vendors

.github/workflows/vendor.yml

@@ -0,0 +1,59 @@
+name: Update Vendor
+
+on:
+  workflow_dispatch:
+  schedule:
+  # At 13:37 UTC every day.
+  - cron: '37 13 * * *'
+
+defaults:
+  run:
+    shell: pwsh
+
+permissions:
+  contents: read
+
+jobs:
+  vendor:
+
+    runs-on: windows-latest
+    continue-on-error: false
+    timeout-minutes: 15
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - id: make-changes
+      run: |
+          $currentVersion = (Get-Content .\vendor\sources.json | ConvertFrom-Json)
+          . .\scripts\update.ps1 -verbose
+          Set-GHVariable -Name COUNT_UPDATED -Value $count
+          $newVersion = (Get-Content .\vendor\sources.json | ConvertFrom-Json)
+          $listUpdated = ""
+          $updateMessage = "| Name | Old Version | New Version |`n| :--- | ---- | ---- |`n"
+          foreach ($s in $newVersion) {
+            $oldVersion = ($currentVersion | Where-Object {$_.name -eq $s.name}).version
+            if ($s.version -ne $oldVersion) {
+              $listUpdated += "$($s.name) v$($s.version), "
+              $updateMessage += "| **$($s.name)** | $oldVersion | **$($s.version)** |`n"
+            }
+          }
+          Set-GHVariable -Name LIST_UPDATED -Value $listUpdated.Trim(', ')
+          echo "UPDATE_MESSAGE<<<EOF`n$updateMessage`n<EOF" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+
+    - uses: peter-evans/create-pull-request@v4
+      with:
+        title: 'Updates to `${{ env.COUNT_UPDATED }}` vendored dependencies'
+        body: |
+          ### Automatically updated `${{ env.COUNT_UPDATED }}` dependencies:
+          ${{ env.UPDATE_MESSAGE }}
+          ---
+          Please verify and then **Merge** the pull request to update.
+        commit-message: 'Update vendored dependencies (${{ env.LIST_UPDATED }})'
+        branch: update-vendor
+        base: master

scripts/build.ps1

@@ -11,11 +11,15 @@
 .EXAMPLE
     .\build.ps1
 
-    Executes the default build for Cmder; Conemu, clink. This is equivalent to the "minimum" style package in the releases
+    Executes the default build for Cmder; ConEmu, clink. This is equivalent to the "minimum" style package in the releases
 .EXAMPLE
     .\build.ps1 -Compile
 
     Recompile the launcher executable if you have the requisite build tools for C++ installed.
+.EXAMPLE
+    .\build.ps1 -Compile -NoVendor
+
+    Skip all downloads and only build launcher.
 .EXAMPLE
     .\build -verbose
 
@@ -49,7 +53,10 @@ Param(
     # Config folder location
     [string]$config = "$PSScriptRoot\..\config",
 
-    # New launcher if you have MSBuild tools installed
+    # Using this option will skip all downloads, if you only need to build launcher
+    [switch]$noVendor,
+
+    # Build launcher if you have MSBuild tools installed
     [switch]$Compile
 )
 
@@ -60,97 +67,113 @@ $cmder_root = Resolve-Path "$PSScriptRoot\.."
 . "$PSScriptRoot\utils.ps1"
 $ErrorActionPreference = "Stop"
 
-Push-Location -Path $saveTo
+if ($Compile) {
-$sources = Get-Content $sourcesPath | Out-String | Convertfrom-Json
+    # Check for requirements
+    Ensure-Executable "msbuild"
 
-# Get the version string
+    # Get the version string
-$version = Get-VersionStr
+    $version = Get-VersionStr
 
-# Check for requirements
-Ensure-Exists $sourcesPath
-Ensure-Executable "7z"
-Ensure-Executable "msbuild"
-New-Item -Type Directory -Path (Join-Path $saveTo "/tmp/") -ErrorAction SilentlyContinue >$null
-
-# Preserve modified (by user) ConEmu setting file
-if ($config -ne "") {
-    $ConEmuXml = Join-Path $saveTo "conemu-maximus5\ConEmu.xml"
-    if (Test-Path $ConEmuXml -pathType leaf) {
-        $ConEmuXmlSave = Join-Path $config "ConEmu.xml"
-        Write-Verbose "Backup '$ConEmuXml' to '$ConEmuXmlSave'"
-        Copy-Item $ConEmuXml $ConEmuXmlSave
-    } else { $ConEmuXml = "" }
-} else { $ConEmuXml = "" }
-
-# Kill ssh-agent.exe if it is running from the $env:cmder_root we are building
-foreach ($ssh_agent in $(get-process ssh-agent -erroraction silentlycontinue)) {
-    if ([string]$($ssh_agent.path) -match [string]$cmder_root.replace('\','\\')) {
-        Write-Verbose $("Stopping " + $ssh_agent.path + "!")
-        Stop-Process $ssh_agent.id
-    }
-}
-
-$vend = $pwd
-foreach ($s in $sources) {
-    Write-Verbose "Getting vendored $($s.name) $($s.version)..."
-
-    # We do not care about the extensions/type of archive
-    $tempArchive = "tmp/$($s.name).tmp"
-    Delete-Existing $tempArchive
-    Delete-Existing $s.name
-
-    Download-File -Url $s.url -File $vend\$tempArchive -ErrorAction Stop
-    Extract-Archive $tempArchive $s.name
-
-    if ((Get-Childitem $s.name).Count -eq 1) {
-        Flatten-Directory($s.name)
-    }
-    # Write current version to .cmderver file, for later.
-    "$($s.version)" | Out-File "$($s.name)/.cmderver"
-}
-
-# Restore user configuration
-if ($ConEmuXml -ne "") {
-    Write-Verbose "Restore '$ConEmuXmlSave' to '$ConEmuXml'"
-    Copy-Item $ConEmuXmlSave $ConEmuXml
-}
-
-Pop-Location
-
-if($Compile) {
     Push-Location -Path $launcher
-    Create-RC $version ($launcher + '\src\version.rc2');
+    Create-RC $version ($launcher + '\src\version.rc2')
-    # https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
+
+    Write-Verbose "Building the launcher..."
+
+    # Referene: https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
     msbuild CmderLauncher.vcxproj /t:Clean,Build /p:configuration=Release /m
+
     if ($LastExitCode -ne 0) {
-        throw "MSBuild failed to build the executable."
+        throw "MSBuild failed to build the launcher executable."
-    }
-    else {
-        Write-Verbose "Successfully built Cmder v$version!"
-        if ( $Env:APPVEYOR -eq 'True' ) {
-            Add-AppveyorMessage -Message "Building Cmder v$version was successful." -Category Information
-        }
-        if ( $Env:GITHUB_ACTIONS -eq 'true' ) {
-            Write-Output "::notice title=Build Complete::Building Cmder v$version was successful."
-        }
     }
     Pop-Location
-} else {
+}
-    Write-Warning "You are not building a launcher, Use -Compile"
+
+if (-Not $noVendor) {
+    # Check for requirements
+    Ensure-Exists $sourcesPath
+    Ensure-Executable "7z"
+
+    # Get the vendor sources
+    $sources = Get-Content $sourcesPath | Out-String | ConvertFrom-Json
+
+    Push-Location -Path $saveTo
+    New-Item -Type Directory -Path (Join-Path $saveTo "/tmp/") -ErrorAction SilentlyContinue >$null
+
+    $vend = $pwd
+
+    # Preserve modified (by user) ConEmu setting file
+    if ($config -ne "") {
+        $ConEmuXml = Join-Path $saveTo "conemu-maximus5\ConEmu.xml"
+        if (Test-Path $ConEmuXml -pathType leaf) {
+            $ConEmuXmlSave = Join-Path $config "ConEmu.xml"
+            Write-Verbose "Backup '$ConEmuXml' to '$ConEmuXmlSave'"
+            Copy-Item $ConEmuXml $ConEmuXmlSave
+        } else { $ConEmuXml = "" }
+    } else { $ConEmuXml = "" }
+
+    # Kill ssh-agent.exe if it is running from the $env:cmder_root we are building
+    foreach ($ssh_agent in $(Get-Process ssh-agent -ErrorAction SilentlyContinue)) {
+        if ([string]$($ssh_agent.path) -Match [string]$cmder_root.replace('\','\\')) {
+            Write-Verbose $("Stopping " + $ssh_agent.path + "!")
+            Stop-Process $ssh_agent.id
+        }
+    }
+
+    foreach ($s in $sources) {
+        Write-Verbose "Getting vendored $($s.name) $($s.version)..."
+
+        # We do not care about the extensions/type of archive
+        $tempArchive = "tmp/$($s.name).tmp"
+        Delete-Existing $tempArchive
+        Delete-Existing $s.name
+
+        Download-File -Url $s.url -File $vend\$tempArchive -ErrorAction Stop
+        Extract-Archive $tempArchive $s.name
+
+        if ((Get-ChildItem $s.name).Count -eq 1) {
+            Flatten-Directory($s.name)
+        }
+
+        # Write current version to .cmderver file, for later.
+        "$($s.version)" | Out-File "$($s.name)/.cmderver"
+    }
+
+    # Restore ConEmu user configuration
+    if ($ConEmuXml -ne "") {
+        Write-Verbose "Restore '$ConEmuXmlSave' to '$ConEmuXml'"
+        Copy-Item $ConEmuXmlSave $ConEmuXml
+    }
+
+    # Put vendor\cmder.sh in /etc/profile.d so it runs when we start bash or mintty
+    if ( (Test-Path $($saveTo + "git-for-windows/etc/profile.d") ) ) {
+        Write-Verbose "Adding cmder.sh /etc/profile.d"
+        Copy-Item $($saveTo + "cmder.sh") $($saveTo + "git-for-windows/etc/profile.d/cmder.sh")
+    }
+
+    # Replace /etc/profile.d/git-prompt.sh with cmder lambda prompt so it runs when we start bash or mintty
+    if ( !(Test-Path $($saveTo + "git-for-windows/etc/profile.d/git-prompt.sh.bak") ) ) {
+        Write-Verbose "Replacing /etc/profile.d/git-prompt.sh with our git-prompt.sh"
+        Move-Item $($saveTo + "git-for-windows/etc/profile.d/git-prompt.sh") $($saveTo + "git-for-windows/etc/profile.d/git-prompt.sh.bak")
+        Copy-Item $($saveTo + "git-prompt.sh") $($saveTo + "git-for-windows/etc/profile.d/git-prompt.sh")
+    }
+
+    Pop-Location
+}
+
+if (-Not $Compile -Or $noVendor) {
+    Write-Warning "You are not building the full project, Use -Compile without -noVendor"
     Write-Warning "This cannot be a release. Test build only!"
+    return
 }
 
-# Put vendor\cmder.sh in /etc/profile.d so it runs when we start bash or mintty
+Write-Verbose "Successfully built Cmder v$version!"
-if ( (Test-Path $($SaveTo + "git-for-windows/etc/profile.d") ) ) {
+
-    Write-Verbose "Adding cmder.sh /etc/profile.d"
+if ( $Env:APPVEYOR -eq 'True' ) {
-    Copy-Item $($SaveTo + "cmder.sh") $($SaveTo + "git-for-windows/etc/profile.d/cmder.sh")
+    Add-AppveyorMessage -Message "Building Cmder v$version was successful." -Category Information
 }
 
-# Replace /etc/profile.d/git-prompt.sh with cmder lambda prompt so it runs when we start bash or mintty
+if ( $Env:GITHUB_ACTIONS -eq 'true' ) {
-if ( !(Test-Path $($SaveTo + "git-for-windows/etc/profile.d/git-prompt.sh.bak") ) ) {
+    Write-Output "::notice title=Build Complete::Building Cmder v$version was successful."
-    Write-Verbose "Replacing /etc/profile.d/git-prompt.sh with our git-prompt.sh"
-    Move-Item $($SaveTo + "git-for-windows/etc/profile.d/git-prompt.sh") $($SaveTo + "git-for-windows/etc/profile.d/git-prompt.sh.bak")
-    Copy-Item $($SaveTo + "git-prompt.sh") $($SaveTo + "git-for-windows/etc/profile.d/git-prompt.sh")
 }
 
 Write-Host -ForegroundColor green "All good and done!"

scripts/pack.ps1

@@ -11,7 +11,7 @@
 
     Creates default archives for cmder
 .EXAMPLE
-    .\build -verbose
+    .\pack.ps1 -verbose
 
     Creates default archives for cmder with plenty of information
 .NOTES
@@ -52,7 +52,7 @@ Push-Location -Path $cmderRoot
 Delete-Existing "$cmderRoot\Version*"
 Delete-Existing "$cmderRoot\build\*"
 
-If(-not (Test-Path -PathType container $saveTo)) {
+if (-not (Test-Path -PathType container $saveTo)) {
     (New-Item -ItemType Directory -Path $saveTo) | Out-Null
 }
 

scripts/update.ps1

@@ -0,0 +1,266 @@
+<#
+.Synopsis
+    Update Cmder vendored dependencies
+.DESCRIPTION
+    This script updates dependencies to the latest version in vendor/sources.json file.
+
+    You will need to make this script executable by setting your Powershell Execution Policy to Remote signed
+    Then unblock the script for execution with UnblockFile .\build.ps1
+.EXAMPLE
+    .\build.ps1
+
+    Updates the dependency sources in the default location, the vendor/sources.json file.
+.EXAMPLE
+    .\build -verbose
+
+    Updates the dependency sources and see what's going on.
+.EXAMPLE
+    .\build.ps1 -SourcesPath '~/custom/vendors.json'
+
+    Specify the path to update dependency sources file at.
+.NOTES
+    AUTHORS
+    David Refoua <David@Refoua.me>
+    Part of the Cmder project.
+.LINK
+    http://cmder.app/ - Project Home
+#>
+[CmdletBinding(SupportsShouldProcess=$true)]
+Param(
+    # CmdletBinding will give us;
+    # -verbose switch to turn on logging and
+    # -whatif switch to not actually make changes
+
+    # Path to the vendor configuration source file
+    [string]$sourcesPath = "$PSScriptRoot\..\vendor\sources.json"
+)
+
+# Get the root directory of the cmder project.
+$cmder_root = Resolve-Path "$PSScriptRoot\.."
+
+# Dot source util functions into this scope
+. "$PSScriptRoot\utils.ps1"
+$ErrorActionPreference = "Stop"
+
+# Attempts to match the current link with the new link, returning the count of matching characters.
+function Match-Filenames {
+    param (
+        $url,
+        $downloadUrl,
+        $fromEnd
+    )
+
+    $filename = [System.IO.Path]::GetFileName($url)
+    $filenameDownload = [System.IO.Path]::GetFileName($downloadUrl)
+
+    $position = 0
+
+    if ([String]::IsNullOrEmpty($filename) -Or [String]::IsNullOrEmpty($filenameDownload)) {
+        throw "Either one or both filenames are empty!"
+    }
+
+    if ($fromEnd) {
+        $arr = $filename -split ""
+        [array]::Reverse($arr)
+        $filename = $arr -join ''
+        $arr = $filenameDownload -split ""
+        [array]::Reverse($arr)
+        $filenameDownload = $arr -join ''
+    }
+
+    while ($filename.Substring($position, 1) -eq $filenameDownload.Substring($position, 1)) {
+        $position++
+
+        if ( ($position -ge $filename.Length) -or ($position -ge $filenameDownload.Length) ) {
+            break
+        }
+    }
+
+    return $position
+}
+
+# Uses the GitHub api in order to fetch the current download links for the latest releases of the repo.
+function Fetch-DownloadUrl {
+    param (
+        [Parameter(Mandatory = $true)]
+        $urlStr
+    )
+
+    $url = [uri] $urlStr
+
+    if ((-Not $url) -Or ($null -eq $url) -Or ($url -eq '')) {
+        throw "Failed to parse url: $urlStr"
+    }
+
+    if (-Not ("http", "https" -Contains $url.Scheme)) {
+        throw "unknown source scheme: $($url.Scheme)"
+    }
+
+    if (-Not ($url.Host -ilike "*github.com")) {
+        throw "unknown source domain: $($url.Host)"
+    }
+
+    $p = $url.Segments.Split([Environment]::NewLine)
+
+    # Api server for GitHub
+    $urlHost = "api.github.com"
+
+    # Path for releases end-point
+    $urlPath = [IO.Path]::Combine('repos', $p[1], $p[2], 'releases').Trim('/')
+
+    $apiUrl = [uri] (New-Object System.UriBuilder -ArgumentList $url.Scheme, $urlHost, -1, $urlPath).Uri
+
+    $info = Invoke-RestMethod -Uri $apiUrl
+
+    $downloadLinks = (New-Object System.Collections.Generic.List[System.Object])
+
+    $charCount = 0
+
+    if (-Not ($info -Is [array])) {
+        throw "The response received from API server is invalid"
+    }
+
+    :loop foreach ($i in $info) {
+        if (-Not ($i.assets -Is [array])) {
+            continue
+        }
+
+        foreach ($a in $i.assets) {
+            if ([String]::IsNullOrEmpty($a.browser_download_url)) {
+                continue
+            }
+
+			# Skip some download links as we're not interested in them
+            if ( $a.browser_download_url -ilike "*_symbols*" ) {
+                continue
+            }
+
+            $score = Match-Filenames $url $a.browser_download_url
+
+			# Skip links that don't match or are less similar
+            if ( ($score -eq 0) -or ($score -lt $charCount) ) {
+                continue
+            }
+
+			# If we reach the same download link as we have
+            if ( $score -eq [System.IO.Path]::GetFileName($url).Length ) {
+            }
+
+            $charCount = $score
+            $downloadLinks.Add($a.browser_download_url)
+        }
+
+		# If at least one download link was found, don't continue with older releases
+        if ( $downloadLinks.Length -gt 0 ) {
+            break :loop
+        }
+    }
+
+	# Special case for archive downloads of repository
+    if (($null -eq $downloadLinks) -Or (-Not $downloadLinks)) {
+        if ((($p | % {$_.Trim('/')}) -Contains "archive") -And $info[0].tag_name) {
+            for ($i = 0; $i -lt $p.Length; $i++) {
+                if ($p[$i].Trim('/') -eq "archive") {
+                    $p[$i+1] = $info[0].tag_name + ".zip"
+                    $downloadLinks = $url.Scheme + "://" + $url.Host + ($p -Join '')
+                    return $downloadLinks
+                }
+            }
+        }
+        return ''
+    }
+
+    $temp = $downloadLinks | Where-Object { (Match-Filenames $url $_) -eq $charCount }
+
+    $downloadLinks = (New-Object System.Collections.Generic.List[System.Object])
+
+    $charCount = 0
+
+    foreach ($l in $temp) {
+        $score = Match-Filenames $url $l true
+
+        if ( ($score -eq 0) -or ($score -lt $charCount) ) {
+            continue
+        }
+
+        $charCount = $score
+    }
+
+    $downloadLinks = $temp | Where-Object { (Match-Filenames $url $_ true) -eq $charCount }
+
+    if (($null -eq $downloadLinks) -Or (-Not $downloadLinks)) {
+        throw "No suitable download links matched for the url!"
+    }
+
+    if (-Not($downloadLinks -is [String])) {
+        throw "Found multiple matches for the same url:`n" + $downloadLinks
+    }
+
+    return $downloadLinks
+}
+
+$count = 0
+
+# Read the current sources content
+$sources = Get-Content $sourcesPath | Out-String | ConvertFrom-Json
+
+foreach ($s in $sources) {
+    Write-Verbose "Updating sources link for $($s.name)..."
+
+	Write-Verbose "Old Link: $($s.url)"
+
+    $downloadUrl = Fetch-DownloadUrl $s.url
+
+    if (($null -eq $downloadUrl) -Or ($downloadUrl -eq '')) {
+        Write-Verbose "No new links were found"
+        continue
+    }
+
+    Write-Verbose "Link: $downloadUrl"
+
+    $url = [uri] $downloadUrl
+
+    $version = ''
+
+    if ( ($url.Segments[-3] -eq "download/") -And ($url.Segments[-2].StartsWith("v")) ) {
+        $version = $url.Segments[-2].TrimStart('v').TrimEnd('/')
+    }
+
+    if ( ($url.Segments[-2] -eq "archive/") ) {
+        $version = [System.IO.Path]::GetFileNameWithoutExtension($url.Segments[-1].TrimStart('v').TrimEnd('/'))
+    }
+
+    if ( $version -eq '' ) {
+        throw "Unable to extract version from url string"
+    }
+
+    Write-Verbose "Version: $version"
+
+	if ( $s.version -ne $version ) {
+		# if ( ([System.Version] $s.version) -gt ([System.Version] $version) ) {
+		# 	throw "The current version $($s.version) is already newer than the found version $version!"
+		# }
+
+		$count++
+	}
+
+    $s.url = $downloadUrl
+    $s.version = $version
+}
+
+$sources | ConvertTo-Json | Set-Content $sourcesPath
+
+if ($count -eq 0) {
+	Write-Host -ForegroundColor yellow "No new releases were found."
+	return
+}
+
+if ( $Env:APPVEYOR -eq 'True' ) {
+    Add-AppveyorMessage -Message "Successfully updated $count dependencies." -Category Information
+}
+
+if ( $Env:GITHUB_ACTIONS -eq 'true' ) {
+    Write-Output "::notice title=Task Complete::Successfully updated $count dependencies."
+}
+
+Write-Host -ForegroundColor green "Successfully updated $count dependencies."

scripts/utils.ps1

@@ -9,13 +9,13 @@ function Ensure-Exists($path) {
 function Ensure-Executable($command) {
     try { Get-Command $command -ErrorAction Stop > $null }
     catch {
-        If( ($command -eq "7z") -and (Test-Path "$env:programfiles\7-zip\7z.exe") ){
+        if( ($command -eq "7z") -and (Test-Path "$env:programfiles\7-zip\7z.exe") ){
             Set-Alias -Name "7z" -Value "$env:programfiles\7-zip\7z.exe" -Scope script
         }
-        ElseIf( ($command -eq "7z") -and (Test-Path "$env:programw6432\7-zip\7z.exe") ) {
+        elseif( ($command -eq "7z") -and (Test-Path "$env:programw6432\7-zip\7z.exe") ) {
             Set-Alias -Name "7z" -Value "$env:programw6432\7-zip\7z.exe" -Scope script
         }
-        Else {
+        else {
             Write-Error "Missing $command! Ensure it is installed and on in the PATH"
             exit 1
         }
@@ -67,7 +67,32 @@ function Digest-Hash($path) {
     return Invoke-Expression "md5sum $path"
 }
 
-function Get-VersionStr() {
+function Set-GHVariable {
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Name,
+        [Parameter(Mandatory = $true)]
+        [string]$Value
+    )
+
+    Write-Verbose "Setting CI variable $Name to $Value" -Verbose
+
+    if ($env:GITHUB_ENV) {
+        echo "$Name=$Value" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+    }
+}
+
+function Get-GHTempPath {
+    $temp = [System.IO.Path]::GetTempPath()
+    if ($env:RUNNER_TEMP) {
+        $temp = $env:RUNNER_TEMP
+    }
+
+    Write-Verbose "Get CI Temp path: $temp" -Verbose
+    return $temp
+}
+
+function Get-VersionStr {
     # Clear existing variable
     if ($string) { Clear-Variable -name string }