Received-SPF: Pass (mailfrom) identity=mailfrom; client-ip=209.85.128.178; helo=mail-yw1-f178.google.com; envelope-from=peter@netlandish.com; receiver=<UNKNOWN> 
From: Peter Sanchez <peter@netlandish.com>
To: ~netlandish/links-dev@lists.code.netlandish.com
Cc: Peter Sanchez <peter@netlandish.com>
Subject: [PATCH links] Add utf8 sanitizer to address edge case import errors.
Date: Wed, 26 Feb 2025 19:03:49 -0600
Message-ID: <20250227010406.32668-1-peter@netlandish.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit

Fixes: https://todo.code.netlandish.com/~netlandish/links/96
Signed-off-by: Peter Sanchez <peter@netlandish.com>
---
 core/import.go |  5 +++--
 helpers.go     | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/core/import.go b/core/import.go
index 1671f47..ae8e14e 100644
--- a/core/import.go
+++ b/core/import.go
@@ -114,6 +114,7 @@ func (p pinBoardObj) GetTags() []string {
 func trimTags(tags []string) []string {
 	var ret []string
 	for _, t := range tags {
+		t = links.SanitizeUTF8(t)
 		if len(t) > 50 {
 			t = t[:50]
 		}
@@ -300,9 +301,9 @@ func processOrgLinks(obj importObj, baseURLMap map[string]int,
 		title = title[:146] + "..."
 	}
 	return &models.OrgLink{
-		Title:       title,
+		Title:       links.SanitizeUTF8(title),
 		URL:         obj.GetURL(),
-		Description: obj.GetDescription(),
+		Description: links.SanitizeUTF8(obj.GetDescription()),
 		BaseURLID:   sql.NullInt64{Valid: true, Int64: int64(baseID)},
 		OrgID:       org.ID,
 		UserID:      int(user.ID),
diff --git a/helpers.go b/helpers.go
index 36a6eb7..125698a 100644
--- a/helpers.go
+++ b/helpers.go
@@ -23,6 +23,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"git.sr.ht/~emersion/gqlclient"
 	"github.com/99designs/gqlgen/graphql"
@@ -1159,3 +1160,23 @@ func IPForContext(ctx context.Context) string {
 	}
 	return ip
 }
+
+// SanitizeUTF8 will strip out invalid utf-8 characters
+func SanitizeUTF8(input string) string {
+	if utf8.ValidString(input) {
+		return input
+	}
+
+	var b strings.Builder
+	for i := 0; i < len(input); {
+		r, size := utf8.DecodeRuneInString(input[i:])
+		if r == utf8.RuneError && size == 1 {
+			// Replace invalid bytes with a space
+			b.WriteString(" ")
+		} else {
+			b.WriteRune(r)
+		}
+		i += size
+	}
+	return b.String()
+}
-- 
2.47.2