links: Removing calling of ParseBaseURL during bulk import v1 APPLIED

Peter Sanchez: 1
 Removing calling of ParseBaseURL during bulk import

 2 files changed, 7 insertions(+), 7 deletions(-)

[PATCH links] Removing calling of ParseBaseURL during bulk import Export this patch

---
This is just to reomve retries when calling ParseBaseURLTask and also
not calling it during import.

This process will be moved to a period job for base urls that have not
been processed. We will add a job to process these.

See: https://todo.code.netlandish.com/~netlandish/links/100

 core/import.go     | 10 +++++-----
 core/processors.go |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/import.go b/core/import.go
index 12a0aac..1797d51 100644
--- a/core/import.go
+++ b/core/import.go
@@ -10,7 +10,6 @@ import (
 	"links/models"
 	"net/url"
 	"strings"
-	"time"
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/labstack/echo/v4"
@@ -274,7 +273,7 @@ func importBaseURLs(ctx context.Context, objAdapter *importAdapter) (map[string]
 			return nil, err
 		}
 		baseURLMap[baseURL.URL] = baseURL.ID
-		srv.QueueTask("import", ParseBaseURLTask(srv, baseURL))
+		//srv.QueueTask("import", ParseBaseURLTask(srv, baseURL))
 	}
 	return baseURLMap, nil
 }
@@ -452,8 +451,6 @@ func ImportFromPinBoard(ctx context.Context, path string,
 				pinBoards:   pinBoardList,
 			}
 
-			totalCount += listlen
-
 			baseURLMap, err := importBaseURLs(ctx, adapter)
 			if err != nil {
 				return err
@@ -470,7 +467,10 @@ func ImportFromPinBoard(ctx context.Context, path string,
 			if err != nil {
 				return err
 			}
-			time.Sleep(3 * time.Second) // Let the parse url workers catch up
+
+			totalCount += listlen
+			fmt.Println("Processed", totalCount, "entries...")
+			//time.Sleep(3 * time.Second) // Let the parse url workers catch up
 		} else {
 			break // No more items to process
 		}
diff --git a/core/processors.go b/core/processors.go
index e6160e6..5720d32 100644
--- a/core/processors.go
+++ b/core/processors.go
@@ -65,7 +65,7 @@ func ImportBookmarksTask(c echo.Context, origin int, path string,
 			server.Logger().Printf("Error sending import complete email: %v", err)
 		}
 		return nil
-	}).Retries(0).Before(func(ctx context.Context, task *work.Task) {
+	}).Before(func(ctx context.Context, task *work.Task) {
 		gobwebs.TaskIDWork(task)
 		gctx.Server.Logger().Printf(
 			"Running task ImportBookmarksTask %s for the %d attempt.",
@@ -90,7 +90,7 @@ func ParseBaseURLTask(srv *server.Server, baseURL *models.BaseURL) *work.Task {
 		ctx = database.Context(ctx, srv.DB)
 		ctx = timezone.Context(ctx, "UTC")
 		return links.ParseBaseURL(ctx, baseURL)
-	}).Retries(0).Before(func(ctx context.Context, task *work.Task) {
+	}).Before(func(ctx context.Context, task *work.Task) {
 		gobwebs.TaskIDWork(task)
 		srv.Logger().Printf(
 			"Running task ParseBaseURLTask %s for the %d attempt.",
-- 
2.47.2

Peter Sanchez <peter@netlandish.com>

27 days ago

Applied.

To git@git.code.netlandish.com:~netlandish/links
   70a5244..4780cdb  master -> master