diff --git a/tools/gitops-migrate/README.md b/tools/gitops-migrate/README.md deleted file mode 100644 index 90c33dce83..0000000000 --- a/tools/gitops-migrate/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# GitOps migration tool - -Fleet 4.74.0 includes [breaking changes](https://github.com/fleetdm/fleet/pull/30837/files#r2205252594) to the [experimental](https://fleetdm.com/handbook/company/product-groups#experimental-features) software YAML files. This tool automatically migrates your YAML to the new YAML format Fleet 4.74.0 expects. - -How to upgrade to 4.74.0: - -1. Update your YAML by running the script documented in this file -2. In your GitOps repo, open a PR with your updated YAML -3. Upgrade Fleet to 4.74.0 -4. Merge in your PR - -## Overview - -This script automates the migration of software configuration keys from individual software packages to fleet-level configurations. It processes YAML files in the `it-and-security/fleets/` directory and moves the following keys from referenced software files to the fleet files: - -- `self_service` -- `categories` -- `labels_include_any` -- `labels_exclude_any` - -## Prerequisites - -**yq** is required (version 4 or higher) - -```bash -# Install on macOS -brew install yq - -# Install on Ubuntu/Debian -# yq installed from apt is NOT supported -sudo snap install yq - -# Install on other systems - see https://github.com/mikefarah/yq -``` - -## Usage - -### Basic usage - -```bash -./tools/gitops-migrate/migrate.sh -``` - -The script will: -1. Automatically discover all `.yml` files in the specified fleets directory -2. For each fleet file, process all packages listed in `software.packages[]` -3. Extract the target keys from each referenced software file (Pass 1) -4. Move those keys to the corresponding package entry in the fleet file (Pass 1) -5. Remove the keys from the original software files after all fleets are processed (Pass 2) - -### What the script does - -#### Before running the script - -**Team file (`it-and-security/fleets/example.yml`):** -```yaml -name: Example Team -software: - packages: - - path: ../lib/macos/software/firefox.yml -``` - -**Software file (`it-and-security/lib/macos/software/firefox.yml`):** - -```yaml -url: https://download.mozilla.org/... -self_service: true -categories: - - "Web Browser" -labels_include_any: - - "Department:Engineering" -labels_exclude_any: - - "OS:Windows" -``` - -#### After running the script - -**Team file (`it-and-security/fleets/example.yml`):** -```yaml -name: Example Team -software: - packages: - - path: ../lib/macos/software/firefox.yml - self_service: true - categories: - - "Web Browser" - labels_include_any: - - "Department:Engineering" - labels_exclude_any: - - "OS:Windows" -``` - -**Software file (`it-and-security/lib/macos/software/firefox.yml`):** - -```yaml -url: https://download.mozilla.org/... -``` - -Example output: -``` -GitOps Migration Tool -Moving keys from software files to fleet files -Teams directory: it-and-security/fleets - -Finding team files... -Found 3 team files - -=== PASS 1: UPDATING TEAM FILES === -Processing team file: it-and-security/teams/workstations.yml - Found 2 packages - Processing package 1/2 - Package path: ../lib/macos/software/mozilla-firefox.yml - Processing: it-and-security/lib/macos/software/mozilla-firefox.yml - Adding keys to team file at package index 0 - Added self_service - Added categories - ✓ Package processed successfully - -=== PASS 2: CLEANING UP SOFTWARE FILES === -Removing keys from 15 unique software files - Removing keys from: mozilla-firefox.yml -✓ Software file cleanup complete - -=== PROCESSING COMPLETE === -Teams processed: 3 -Packages processed: 8 -✓ All files processed successfully! -``` - - - -## Troubleshooting - -### Common issues - -1. **"yq is required but not installed"** - - Install yq using the instructions in Prerequisites - -2. **"yq version 4 or higher is required"** - - Upgrade yq: `brew upgrade yq` - -3. **"Teams directory not found"** - - Verify the directory path argument is correct - - Ensure you're running from the correct location - -4. **"Software file not found"** - - Check that the `path` in the fleet file is correct relative to the fleet file location - -### Debug mode - -For troubleshooting, you can add debug output by modifying the script temporarily: -```bash -# Add this after the shebang line -set -euxo pipefail # Adds debug output -``` - -## Contributing - -When modifying this tool: -1. Test on a small subset of files first -2. Ensure shellcheck passes: `shellcheck migrate.sh` -3. Verify YAML syntax validation works correctly -4. Test the two-pass processing logic thoroughly diff --git a/tools/gitops-migrate/migrate.sh b/tools/gitops-migrate/migrate.sh deleted file mode 100755 index 6ee60794ca..0000000000 --- a/tools/gitops-migrate/migrate.sh +++ /dev/null @@ -1,335 +0,0 @@ -#!/bin/bash - -# GitOps Migration Tool -# Moves self_service, categories, labels_exclude_any, labels_include_any keys -# from software YAML files to fleet (formerly "teams") YAML files -# -# Usage: ./migrate.sh -# Example: ./migrate.sh it-and-security/teams - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Global counters -PROCESSED_TEAMS=0 -PROCESSED_PACKAGES=0 -ERRORS=0 - -# Array to track software files that have been processed -PROCESSED_SOFTWARE_FILES=() - -# Check if yq is installed -check_dependencies() { - if ! command -v yq &> /dev/null; then - echo -e "${RED}Error: yq is required but not installed. Please install yq first.${NC}" - echo "Install with: brew install yq" - exit 1 - fi - - # Check yq version (we need v4+) - YQ_VERSION=$(yq --version | cut -d' ' -f4 | cut -d'v' -f2 | cut -d'.' -f1) - if [[ "$YQ_VERSION" == "" ]]; then - YQ_VERSION="0" - fi - if [ "$YQ_VERSION" -lt 4 ]; then - echo -e "${RED}Error: yq version 4 or higher is required${NC}" - exit 1 - fi -} - -# Show usage information -show_usage() { - echo "Usage: $0 " - echo - echo "Process YAML files in the specified teams directory, moving keys from" - echo "referenced software files to the team files." - echo - echo "Arguments:" - echo " teams_directory_path Path to directory containing team YAML files" - echo - echo "Examples:" - echo " $0 it-and-security/teams" - echo " $0 /path/to/teams" - echo - echo "Keys moved: self_service, categories, labels_include_any, labels_exclude_any" -} - -# Validate YAML syntax -validate_yaml() { - local file="$1" - if ! yq eval '.' "$file" >/dev/null 2>&1; then - echo -e "${RED}Error: Invalid YAML syntax in $file${NC}" - return 1 - fi - return 0 -} - -# Extract target keys from software file -extract_keys_from_software() { - local software_file="$1" - local temp_file=$(mktemp -p .) - chmod 666 $temp_file - - # Extract the keys we need - { - echo "# Extracted keys from $software_file" - yq eval 'pick(["self_service", "categories", "labels_include_any", "labels_exclude_any"])' "$software_file" 2>/dev/null || echo "{}" - } > "$temp_file" - - echo "$temp_file" -} - -# Remove target keys from software file -remove_keys_from_software() { - local software_file="$1" - - echo -e "${BLUE} Removing keys from: $software_file${NC}" - - # Create a temporary file with keys removed - local temp_file=$(mktemp -p .) - chmod 666 $temp_file - yq eval --output-format=yaml 'del(.self_service, .categories, .labels_include_any, .labels_exclude_any)' "$software_file" > "$temp_file" - - # Replace the original file - mv "$temp_file" "$software_file" -} - -# Add keys to team file at specific package index -add_keys_to_team_file() { - local team_file="$1" - local package_index="$2" - local keys_file="$3" - - # Check if keys file has any meaningful content - if ! yq eval 'keys | length > 0' "$keys_file" >/dev/null 2>&1; then - echo -e "${YELLOW} No keys to move${NC}" - return 0 - fi - - echo -e "${BLUE} Adding keys to team file at package index $package_index${NC}" - - - # Process each key type directly on the team file to preserve formatting - for key in "self_service" "categories" "labels_include_any" "labels_exclude_any"; do - if yq eval "has(\"$key\")" "$keys_file" | grep -q "true"; then - # Use yq to properly extract and merge the value, preserving arrays and complex structures - if yq eval ".$key != null" "$keys_file" | grep -q "true"; then - yq eval -i ".software.packages[$package_index].$key = load(\"$keys_file\").$key" "$team_file" - echo -e "${GREEN} Added $key${NC}" - fi - fi - done -} - -# Process a single team file (Pass 1: Add keys to team files only) -process_team_file() { - local team_file="$1" - echo -e "${GREEN}Processing team file: $team_file${NC}" - - # Check if file has software.packages section - if ! yq eval 'has("software") and .software | has("packages")' "$team_file" | grep -q "true"; then - echo -e "${YELLOW} No software.packages section found, skipping${NC}" - return 0 - fi - - # Get the number of packages - local package_count=$(yq eval '.software.packages | length' "$team_file") - echo -e "${BLUE} Found $package_count packages${NC}" - - # Process each package - for ((i=0; i/dev/null || echo "$software_file") - - if [ ! -f "$software_file" ]; then - echo -e "${RED} Error: Software file not found: $software_file${NC}" - ERRORS=$((ERRORS+1)) - continue - fi - - # Validate software file - if ! validate_yaml "$software_file"; then - echo -e "${RED} Error: Invalid YAML in software file${NC}" - ERRORS=$((ERRORS+1)) - continue - fi - - echo -e "${BLUE} Processing: $software_file${NC}" - - # Extract keys from software file - local keys_temp_file=$(extract_keys_from_software "$software_file") - - # Add keys to team file - add_keys_to_team_file "$team_file" "$i" "$keys_temp_file" - - # Track this software file for cleanup in pass 2 - PROCESSED_SOFTWARE_FILES+=("$software_file") - - # Clean up temp file - rm -f "$keys_temp_file" - - PROCESSED_PACKAGED=$((PROCESSED_PACKAGES+1)) - echo -e "${GREEN} ✓ Package processed successfully${NC}" - done - - # Validate the modified team file - if ! validate_yaml "$team_file"; then - echo -e "${RED} Error: Team file became invalid after processing${NC}" - ERRORS=$((ERRORS+1)) - return 1 - fi - - PROCESSED_TEAMS=$((PROCESSED_TEAMS+1)) - echo -e "${GREEN}✓ Team file processed successfully${NC}" - echo -} - -# Pass 2: Remove keys from all processed software files -cleanup_software_files() { - echo -e "${GREEN}=== PASS 2: CLEANING UP SOFTWARE FILES ===${NC}" - - # Remove duplicates from the array - local unique_files=($(printf "%s\n" "${PROCESSED_SOFTWARE_FILES[@]}" | sort -u)) - - echo -e "${BLUE}Removing keys from ${#unique_files[@]} unique software files${NC}" - - for software_file in "${unique_files[@]}"; do - echo -e "${BLUE} Removing keys from: $software_file${NC}" - remove_keys_from_software "$software_file" - done - - echo -e "${GREEN}✓ Software file cleanup complete${NC}" - echo -} -# Fix Unicode escape sequences back to emoji characters -fix_unicode_emojis() { - local file="$1" - echo -e "${BLUE} Restoring emoji characters in: $(basename "$file")${NC}" - - # Use perl to convert Unicode escape sequences back to actual characters - if command -v perl &> /dev/null; then - perl -i -pe 's/\\U([0-9A-F]{8})/chr(hex($1))/ge' "$file" - else - # Fallback: convert specific known emojis manually - sed -i '' 's/\\U0001F4BB/💻/g' "$file" 2>/dev/null || sed -i 's/\\U0001F4BB/💻/g' "$file" - sed -i '' 's/\\U0001F423/🐣/g' "$file" 2>/dev/null || sed -i 's/\\U0001F423/🐣/g' "$file" - fi -} - -# Fix emojis in all processed team files -restore_emojis_in_team_files() { - echo -e "${GREEN}=== RESTORING EMOJI CHARACTERS ===${NC}" - - for team_file in "${team_files[@]}"; do - if [ -f "$team_file" ] && grep -q "\\\\U[0-9A-F]" "$team_file"; then - fix_unicode_emojis "$team_file" - fi - done - - echo -e "${GREEN}✓ Emoji restoration complete${NC}" - echo -} - - -# Main function -main() { - # Check if directory path argument is provided - if [ $# -eq 0 ]; then - echo -e "${RED}Error: No teams directory path provided${NC}" - echo - show_usage - exit 1 - fi - - # Handle help flags - if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - show_usage - exit 0 - fi - - local teams_dir="$1" - - echo -e "${GREEN}GitOps Migration Tool${NC}" - echo -e "${BLUE}Moving keys from software files to team files${NC}" - echo -e "${BLUE}Teams directory: $teams_dir${NC}" - echo - - # Check dependencies - check_dependencies - - # Check if teams directory exists - if [ ! -d "$teams_dir" ]; then - echo -e "${RED}Error: Teams directory not found: $teams_dir${NC}" - echo "Please provide a valid directory path" - exit 1 - fi - - # Process all YAML files in teams directory - echo -e "${BLUE}Finding team files...${NC}" - - # Use nullglob to handle case where no files match the pattern - shopt -s nullglob - local team_files=("$teams_dir"/*.yml) - shopt -u nullglob - - if [ ${#team_files[@]} -eq 0 ]; then - echo -e "${RED}Error: No YAML files found in teams directory${NC}" - exit 1 - fi - - echo -e "${BLUE}Found ${#team_files[@]} team files${NC}" - echo - - # PASS 1: Process each team file (add keys to team files) - echo -e "${GREEN}=== PASS 1: UPDATING TEAM FILES ===${NC}" - for team_file in "${team_files[@]}"; do - if [ -f "$team_file" ]; then - process_team_file "$team_file" - fi - done - - # PASS 2: Clean up software files (remove keys from software files) - if [ ${#PROCESSED_SOFTWARE_FILES[@]} -gt 0 ]; then - cleanup_software_files - fi - - # PASS 3: Restore emoji characters that may have been converted to Unicode escape sequences - restore_emojis_in_team_files - - # Summary - echo -e "${GREEN}=== PROCESSING COMPLETE ===${NC}" - echo -e "${GREEN}Teams processed: $PROCESSED_TEAMS${NC}" - echo -e "${GREEN}Packages processed: $PROCESSED_PACKAGES${NC}" - if [ $ERRORS -gt 0 ]; then - echo -e "${RED}Errors encountered: $ERRORS${NC}" - echo -e "${YELLOW}Check the output above for details${NC}" - else - echo -e "${GREEN}✓ All files processed successfully!${NC}" - fi - -} - -# Run main function with all script arguments -main "$@"