add migration code to download from one instance and upload to another

cmd/integrationtest/command.go

@@ -44,7 +44,7 @@ func NewIntegrationTestCmd() *cobra.Command {
 			if len(args) >= 5 {
 				metaCheckRepo = args[4]
 			}
-			integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl)
+			integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl, "")
 		},
 	}
 

cmd/migrate/command.go

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (C) 2021-2023 Red Hat, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+ package migrate
+
+ import (
+	 "fmt"
+	 "os"
+
+	 "github.com/commonjava/indy-tests/pkg/integrationtest"
+	 "github.com/spf13/cobra"
+ )
+
+ func NewMigrateCmd() *cobra.Command {
+
+	 exec := &cobra.Command{
+		 Use:     "migrate $indyBaseUrl $datasetRepoUrl $buildId $migrateTargetIndy --dryRun(optional)",
+		 Short:   "To migrate artifact",
+		 Example: "integrationtest http://indy.xyz.com https://gitlab.xyz.com/nos/nos-integrationtest-dataset 2836 test-builds",
+		 Run: func(cmd *cobra.Command, args []string) {
+			 if !validate(args) {
+				 cmd.Help()
+				 os.Exit(1)
+			 }
+			 clearCache, _ := cmd.Flags().GetBool("clearCache")
+			 dryRun, _ := cmd.Flags().GetBool("dryRun")
+			 keepPod, _ := cmd.Flags().GetBool("keepPod")
+			 promoteTargetStore := "pnc-builds"
+			 integrationtest.Run(args[0], args[1], args[2], promoteTargetStore, "", clearCache, dryRun, keepPod, false, "", args[3])
+		 },
+	 }
+
+	 exec.Flags().BoolP("clearCache", "c", false, "Clear cached built artifact files. This will force download from origin again.")
+	 exec.Flags().BoolP("dryRun", "d", false, "Print msg for repo creation, down/upload, promote, and clean up, without really doing it.")
+	 exec.Flags().BoolP("keepPod", "k", false, "Keep the pod after test to debug.")
+	 return exec
+ }
+
+ func validate(args []string) bool {
+	 if len(args) < 4 {
+		 fmt.Printf("There are 4 mandatory arguments: indyBaseUrl, datasetRepoUrl, buildId, migrateTargetIndy!\n")
+		 return false
+	 }
+	 return true
+ }
+

cmd/root/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/commonjava/indy-tests/cmd/integrationtest"
 	"github.com/commonjava/indy-tests/cmd/promotetest"
 	"github.com/commonjava/indy-tests/cmd/statictest"
+	"github.com/commonjava/indy-tests/cmd/migrate"
 	"github.com/spf13/cobra"
 )
 
@@ -29,6 +30,7 @@ func main() {
 	rootCmd.AddCommand(integrationtest.NewIntegrationTestCmd())
 	rootCmd.AddCommand(event.NewEventTestCmd())
 	rootCmd.AddCommand(statictest.NewStaticTestCmd())
+	rootCmd.AddCommand(migrate.NewMigrateCmd())
 
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)

pkg/buildtest/run.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"time"
 
 	common "github.com/commonjava/indy-tests/pkg/common"
 )
@@ -16,24 +17,31 @@ const (
 	PROXY_           = "proxy-"
 )
 
+const DATA_TIME = "2006-01-02 15:04:05"
+
 func Run(originalIndy, foloId, replacement, targetIndy, packageType string, processNum int) {
 	origIndy := originalIndy
 	if !strings.HasPrefix(origIndy, "http://") {
 		origIndy = "http://" + origIndy
 	}
 	foloTrackContent := common.GetFoloRecord(origIndy, foloId)
 	newBuildName := common.GenerateRandomBuildName()
-	DoRun(originalIndy, targetIndy, "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false)
+	DoRun(originalIndy, targetIndy, "", "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false)
 }
 
 // Create the repo structure and do the download/upload
-func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName string, foloTrackContent common.TrackedContent,
+func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageType, newBuildName string, foloTrackContent common.TrackedContent,
 	additionalRepos []string,
 	processNum int, clearCache, dryRun bool) bool {
 
 	common.ValidateTargetIndyOrExit(originalIndy)
 	targetIndyHost, _ := common.ValidateTargetIndyOrExit(targetIndy)
 
+	migrateEnabled := (migrateTargetIndy != "")
+	if migrateEnabled {
+		migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy)
+		fmt.Printf("Migrate to host %s", migrateTargetIndyHost)
+	}
 	// Prepare the indy repos for the whole testing
 	buildMeta := decideMeta(packageType)
 	if !prepareIndyRepos("http://"+targetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) {
@@ -64,8 +72,49 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str
 		}
 		return success
 	}
+
+	migrateFunc := func(md5str, targetArtiURL, migrateTargetArtiURL string) bool {
+		fileLoc := path.Join(downloadDir, path.Base(targetArtiURL))
+		if dryRun {
+			fmt.Printf("Dry run download, url: %s\n", targetArtiURL)
+			return true
+		}
+		success := false
+		success, _ = common.DownloadFile(targetArtiURL, fileLoc)
+		if success {
+			common.Md5Check(fileLoc, md5str)
+			if dryRun {
+				fmt.Printf("Dry run upload, url: %s\n", migrateTargetArtiURL)
+				return true
+			}
+			common.UploadFile(migrateTargetArtiURL, fileLoc)
+		}
+		return success
+	}
+
 	broken := false
-	if len(downloads) > 0 {
+
+	if migrateEnabled {
+		migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy)
+		migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, migrateTargetIndyHost, packageType, newBuildName, foloTrackContent)
+		fmt.Printf("Waiting 60s...\n")
+		time.Sleep(120 * time.Second) // wait for Indy event handled
+		for _, down := range migrateArtifacts {
+			broken = !migrateFunc(down[0], down[1], down[2])
+			if broken {
+				break
+			}
+		}
+		fmt.Println("==========================================")
+		if broken {
+			fmt.Printf("Build test failed due to some downloading errors. Please see above logs to see the details.\n\n")
+			os.Exit(1)
+		}
+		fmt.Printf("Migration artifacts handling finished.\n\n")
+		return true
+	}
+
+	if len(downloads) > 0 && !migrateEnabled {
 		fmt.Println("Start handling downloads artifacts.")
 		fmt.Printf("==========================================\n\n")
 		if processNum > 1 {
@@ -109,7 +158,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str
 
 	uploads := prepareUploadEntriesByFolo(originalIndy, targetIndy, newBuildName, foloTrackContent)
 
-	if len(uploads) > 0 {
+	if len(uploads) > 0 && !migrateEnabled {
 		fmt.Println("Start handling uploads artifacts.")
 		fmt.Printf("==========================================\n\n")
 		if processNum > 1 {
@@ -204,6 +253,61 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string,
 	return result
 }
 
+func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageType,
+	newBuildId string, foloRecord common.TrackedContent) map[string][]string {
+	targetIndy := normIndyURL(targetIndyURL)
+	result := make(map[string][]string)
+	for _, down := range foloRecord.Downloads {
+		var p string
+		downUrl := ""
+		repoPath := strings.ReplaceAll(down.StoreKey, ":", "/")
+		if down.AccessChannel == "GENERIC_PROXY" {
+			repoPath = strings.Replace(repoPath, "generic-http/remote/r-", "generic-http/hosted/h-", 1)
+			p = path.Join("api/content", repoPath, down.Path)
+		} else {
+			if !strings.HasPrefix(down.StoreKey, packageType) {
+				p = path.Join("api/content", repoPath, down.Path)
+			} else {
+				p = path.Join("api/content", packageType, "group", newBuildId, down.Path)
+			}
+		}
+
+		downUrl = fmt.Sprintf("%s%s", targetIndy, p)
+
+		broken := false
+		migratePath := setHostname(down.LocalUrl, migrateTargetIndyHost)
+		fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath)
+		broken = !delArtifact(migratePath)
+		time.Sleep(100 * time.Millisecond)
+
+		if !strings.HasSuffix( down.StoreKey, ":hosted:shared-imports" ) {
+			extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path)
+			fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra)
+			broken = !delArtifact(extra)
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		if down.StoreKey == "npm:remote:npmjs" || down.StoreKey == "maven:remote:central" {
+			migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path)
+			fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath)
+			broken = !delArtifact(migratePath)
+			time.Sleep(100 * time.Millisecond)
+		} else if down.StoreKey == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down.StoreKey, "maven:hosted:build-") {
+			migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down.Path)
+			fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath)
+			broken = !delArtifact(migratePath)
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		if broken {
+			fmt.Printf("[%s] Deletion failed for %s\n", time.Now().Format(DATA_TIME), migratePath)
+		}
+
+		result[down.Path] = []string{down.Md5, downUrl, migratePath}
+	}
+	return result
+}
+
 // For uploads entries, firstly they should be downloaded from original indy server. We use original indy server to
 // make the download url, and use the target indy server to make the upload url
 func prepareUploadEntriesByFolo(originalIndyURL, targetIndyURL, newBuildId string, foloRecord common.TrackedContent) map[string][]string {
@@ -271,3 +375,17 @@ func prepareDownUploadDirectories(buildId string, clearCache bool) (string, stri
 	fmt.Printf("Prepared download dir: %s, upload dir: %s\n", downloadDir, uploadDir)
 	return downloadDir, uploadDir
 }
+
+func setHostname(addr, hostname string) string {
+	u, err := url.Parse(addr)
+	if err != nil {
+		return ""
+	}
+	u.Host = hostname
+	return u.String()
+}
+
+func delArtifact(url string) bool {
+	_, _, succeeded := common.HTTPRequest(url, common.MethodDelete, nil, false, nil, nil, "", false)
+	return succeeded
+}

pkg/integrationtest/run.go

@@ -56,7 +56,7 @@ const (
  * j. Retrieve the metadata files from step #f again, check that the new version is gone
  * k. Clean up. Delete the build group G and the hosted repo A. Delete folo record.
  */
-func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string) {
+func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string, migrateTargetIndy string) {
 	if indyProxyUrl != "" {
 		fmt.Println("Enable generic proxy: " + indyProxyUrl)
 	}
@@ -92,68 +92,71 @@ func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo
 	originalIndy := getOriginalIndyBaseUrl(foloTrackContent.Uploads[0].LocalUrl)
 	buildName := common.GenerateRandomBuildName()
 	prev := t
-	buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun)
+	buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, migrateTargetIndy, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun)
 	t = time.Now()
 	fmt.Printf("Create mock group(%s) and download/upload SUCCESS, elapsed(s): %f\n", buildName, t.Sub(prev).Seconds())
 
 	//k. Delete the temp group and the hosted repo, and folo record
 	defer cleanUp(indyBaseUrl, packageType, buildName, dryRun)
 
-	// Advanced checks
-	if buildSuccess && !dryRun {
-		if !verifyFoloRecord(indyBaseUrl, buildName, foloTrackContent) {
-			return
+	migrateEnabled := (migrateTargetIndy != "")
+	if !migrateEnabled {
+		// Advanced checks
+		if buildSuccess && !dryRun {
+			if !verifyFoloRecord(indyBaseUrl, buildName, foloTrackContent) {
+				return
+			}
 		}
-	}
 
-	//f. Retrieve the metadata files which will be affected by promotion
-	metaFiles := calculateMetadataFiles(foloTrackContent)
-	metaFilesLoc := path.Join(TMP_METADATA_DIR, "before-promote")
-	newVersionNum := buildName[len(common.BUILD_TEST_):]
-	exists := true
-	passed, e := retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists)
-	if !passed {
-		logger.Infof("Metadata validate failed (before). Errors: %s", e.Error())
-		panic("Metadata validate failed")
-	}
-	fmt.Printf("Metadata validate (before) SUCCESS\n")
-
-	//g. Promote the files in hosted repo A to hosted repo pnc-builds
-	foloTrackId := buildName
-	sourceStore, targetStore := getPromotionSrcTargetStores(packageType, buildName, promoteTargetStore, foloTrackContent)
-	resp, _, success := promotetest.DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum, foloTrackContent, dryRun)
-	if !success {
-		fmt.Printf("Promote failed, %s\n", resp)
-		panic("Promote failed")
-	}
+		//f. Retrieve the metadata files which will be affected by promotion
+		metaFiles := calculateMetadataFiles(foloTrackContent)
+		metaFilesLoc := path.Join(TMP_METADATA_DIR, "before-promote")
+		newVersionNum := buildName[len(common.BUILD_TEST_):]
+		exists := true
+		passed, e := retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists)
+		if !passed {
+			logger.Infof("Metadata validate failed (before). Errors: %s", e.Error())
+			panic("Metadata validate failed")
+		}
+		fmt.Printf("Metadata validate (before) SUCCESS\n")
+
+		//g. Promote the files in hosted repo A to hosted repo pnc-builds
+		foloTrackId := buildName
+		sourceStore, targetStore := getPromotionSrcTargetStores(packageType, buildName, promoteTargetStore, foloTrackContent)
+		resp, _, success := promotetest.DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum, foloTrackContent, dryRun)
+		if !success {
+			fmt.Printf("Promote failed, %s\n", resp)
+			panic("Promote failed")
+		}
 
-	//h. Retrieve the metadata files again, check the new version
-	fmt.Printf("Waiting 30s...\n")
-	time.Sleep(30 * time.Second) // wait for Indy event handled
+		//h. Retrieve the metadata files again, check the new version
+		fmt.Printf("Waiting 30s...\n")
+		time.Sleep(30 * time.Second) // wait for Indy event handled
 
-	metaFilesLoc = path.Join(TMP_METADATA_DIR, "after-promote")
-	passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, exists)
-	if !passed {
-		logger.Infof("Metadata validate failed (after promotion). Errors: %s", e.Error())
-		panic("Metadata validate failed")
-	}
-	fmt.Printf("Metadata validate (after promotion) SUCCESS\n")
+		metaFilesLoc = path.Join(TMP_METADATA_DIR, "after-promote")
+		passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, exists)
+		if !passed {
+			logger.Infof("Metadata validate failed (after promotion). Errors: %s", e.Error())
+			panic("Metadata validate failed")
+		}
+		fmt.Printf("Metadata validate (after promotion) SUCCESS\n")
 
-	//i. Rollback the promotion
-	fmt.Printf("Rollback:\n%s\n", resp)
-	promotetest.Rollback(indyBaseUrl, resp, dryRun)
+		//i. Rollback the promotion
+		fmt.Printf("Rollback:\n%s\n", resp)
+		promotetest.Rollback(indyBaseUrl, resp, dryRun)
 
-	//h. Retrieve the metadata files again, check the new version is GONE
-	fmt.Printf("Waiting 30s...\n")
-	time.Sleep(30 * time.Second)
+		//h. Retrieve the metadata files again, check the new version is GONE
+		fmt.Printf("Waiting 30s...\n")
+		time.Sleep(30 * time.Second)
 
-	metaFilesLoc = path.Join(TMP_METADATA_DIR, "rollback")
-	passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists)
-	if !passed {
-		logger.Infof("Metadata validate failed (rollback). Errors: %s", e.Error())
-		panic("Metadata validate failed")
+		metaFilesLoc = path.Join(TMP_METADATA_DIR, "rollback")
+		passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists)
+		if !passed {
+			logger.Infof("Metadata validate failed (rollback). Errors: %s", e.Error())
+			panic("Metadata validate failed")
+		}
+		fmt.Printf("Metadata validate (rollback) SUCCESS\n")
 	}
-	fmt.Printf("Metadata validate (rollback) SUCCESS\n")
 
 	// Pause and keep pod for debugging
 	if keepPod {