Rework the data model to improve the performance

Signed-off-by: Max Magorsch <arzano@gentoo.org>

13 files changed, 312 insertions, 238 deletions

diff --git a/pkg/app/home/home.go b/pkg/app/home/home.go
index e2d3955..fe8a56e 100644
--- a/pkg/app/home/home.go
+++ b/pkg/app/home/home.go
@@ -21,17 +21,17 @@ func Show(w http.ResponseWriter, r *http.Request) {
 		var messages []*models.Message
 		database.DBCon.Model(&messages).
 			WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-				q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + mailingList[0] + `]%'`).
-					WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + mailingList[0] + `]%'`)
-				return q, nil
-			}).
-			WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-				q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
-					WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
-					WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`).
-					WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`)
+				q = q.WhereOr(`subject LIKE '[` + mailingList[0] + `]%'`).
+					WhereOr(`subject LIKE 'Re: [` + mailingList[0] + `]%'`)
 				return q, nil
 			}).
+			//WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			//	q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+			//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+			//		WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`).
+			//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`)
+			//	return q, nil
+			//}).
 			Order("date DESC").
 			Limit(5).
 			Select()
@@ -57,7 +57,7 @@ func Show(w http.ResponseWriter, r *http.Request) {
 
 	templateData := struct {
 		MailingLists   []models.MailingList
-		PopularThreads models.Threads
+		PopularThreads []*models.Message
 		MessageCount   string
 		CurrentMonth   string
 	}{
diff --git a/pkg/app/home/utils.go b/pkg/app/home/utils.go
index f854549..767b60d 100644
--- a/pkg/app/home/utils.go
+++ b/pkg/app/home/utils.go
@@ -19,7 +19,7 @@ func renderIndexTemplate(w http.ResponseWriter, templateData interface{}) {
 				Funcs(template.FuncMap{
 					"makeMessage": func(headers map[string][]string) models.Message {
 						return models.Message{
-							Headers: headers,
+							//Headers: headers,
 						}
 					},
 				}).
@@ -35,7 +35,7 @@ func getAllMessagesCount() int {
 	var messsageCount int
 	database.DBCon.Model((*models.Message)(nil)).QueryOne(pg.Scan(&messsageCount), `
 		SELECT
-			count(DISTINCT messages.headers->>'Message-Id')
+			count(DISTINCT messages.message_id)
 		FROM
 			messages;
 	`)
diff --git a/pkg/app/list/messages.go b/pkg/app/list/messages.go
index 383e891..7a78811 100644
--- a/pkg/app/list/messages.go
+++ b/pkg/app/list/messages.go
@@ -33,20 +33,20 @@ func Messages(w http.ResponseWriter, r *http.Request) {
 
 	var messages []*models.Message
 	query := database.DBCon.Model(&messages).
-		Column("id", "headers", "date").
+		Column("id", "subject", "from", "date").
 		Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
 		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
-				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
-			return q, nil
-		}).
-		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			q = q.WhereOr(`subject LIKE '[` + listName + `]%'`).
+				WhereOr(`subject LIKE 'Re: [` + listName + `]%'`)
 			return q, nil
 		}).
+		//WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+		//	q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+		//	return q, nil
+		//}).
 		Order("date DESC")
 
 	messagesCount, _ := query.Count()
diff --git a/pkg/app/list/show.go b/pkg/app/list/show.go
index 8db8778..d90f236 100644
--- a/pkg/app/list/show.go
+++ b/pkg/app/list/show.go
@@ -18,17 +18,17 @@ func Show(w http.ResponseWriter, r *http.Request) {
 	}
 	err := database.DBCon.Model((*models.Message)(nil)).
 		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
-				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
-			return q, nil
-		}).
-		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			q = q.WhereOr(`subject LIKE '[` + listName + `]%'`).
+				WhereOr(`subject LIKE 'Re: [` + listName + `]%'`)
 			return q, nil
 		}).
+		//WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+		//	q = q.WhereOr(`to LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`cc LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`to LIKE '%` + listName + `@gentoo.org%'`).
+		//		WhereOr(`cc LIKE '%` + listName + `@gentoo.org%'`)
+		//	return q, nil
+		//}).
 		ColumnExpr("to_char(date, 'YYYY-MM') AS combined_date").
 		ColumnExpr("count(*) AS message_count").
 		Group("combined_date").
diff --git a/pkg/app/list/threads.go b/pkg/app/list/threads.go
index 33ade3c..c069673 100644
--- a/pkg/app/list/threads.go
+++ b/pkg/app/list/threads.go
@@ -32,22 +32,21 @@ func Threads(w http.ResponseWriter, r *http.Request) {
 
 	var messages []*models.Message
 	query := database.DBCon.Model(&messages).
-		Column("id", "headers", "date").
+		Column("id", "subject", "from", "date").
 		Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
-		Where(`NOT headers::jsonb ? 'References'`).
-		Where(`NOT headers::jsonb ? 'In-Reply-To'`).
+		Where(`starts_thread = TRUE`).
 		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
-				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
-			return q, nil
-		}).
-		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
-			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
-				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			q = q.WhereOr(`subject LIKE '[` + listName + `]%'`).
+				WhereOr(`subject LIKE 'Re: [` + listName + `]%'`)
 			return q, nil
 		}).
+		//WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+		//	q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+		//		WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+		//	return q, nil
+		//}).
 		Order("date DESC")
 
 	messagesCount, _ := query.Count()
diff --git a/pkg/app/message/show.go b/pkg/app/message/show.go
index a027596..3d56544 100644
--- a/pkg/app/message/show.go
+++ b/pkg/app/message/show.go
@@ -26,9 +26,9 @@ func Show(w http.ResponseWriter, r *http.Request) {
 
 	var inReplyTos []*models.Message
 	var inReplyTo *models.Message
-	if message.HasHeaderField("In-Reply-To") {
+	if message.InReplyTo != nil {
 		err = database.DBCon.Model(&inReplyTos).
-			Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.GetHeaderField("In-Reply-To") + `'`).
+			Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.InReplyTo.Id + `'`).
 			Select()
 		if err != nil || len(inReplyTos) < 1 {
 			inReplyTo = nil
@@ -41,8 +41,8 @@ func Show(w http.ResponseWriter, r *http.Request) {
 
 	var replies []*models.Message
 	database.DBCon.Model(&replies).
-		Where(`(headers::jsonb->>'References')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
-		WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
+		Where(`(headers::jsonb->>'References')::jsonb ? '` + message.Id + `'`).
+		WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.Id + `'`).
 		Order("date ASC").Select()
 
 	renderMessageTemplate(w, listName, message, inReplyTo, replies)
diff --git a/pkg/app/message/utils.go b/pkg/app/message/utils.go
index 0cb40f5..f79e6f3 100644
--- a/pkg/app/message/utils.go
+++ b/pkg/app/message/utils.go
@@ -35,29 +35,40 @@ func renderMessageTemplate(w http.ResponseWriter, listName string, message *mode
 
 func getFuncMap() template.FuncMap {
 	return template.FuncMap{
-		"formatAddr": func(addr string) string {
-			if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") {
-				addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o")
-				addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o")
-			} else {
-				start := false
-				for i := len(addr) - 1; i > 0; i-- {
-					if addr[i] == '@' {
-						break
-					}
-					if start {
-						out := []rune(addr)
-						out[i] = '×'
-						addr = string(out)
-					}
-					if addr[i] == '.' {
-						start = true
-					}
-				}
+		"formatAddr": formatAddr,
+		"formatAddrList": formatAddrList,
+	}
+}
+
+func formatAddr(addr string) string {
+	if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") {
+		addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o")
+		addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o")
+	} else {
+		start := false
+		for i := len(addr) - 1; i > 0; i-- {
+			if addr[i] == '@' {
+				break
+			}
+			if start {
+				out := []rune(addr)
+				out[i] = '×'
+				addr = string(out)
 			}
-			return addr
-		},
+			if addr[i] == '.' {
+				start = true
+			}
+		}
+	}
+	return addr
+}
+
+func formatAddrList(addrList []string) string {
+	var formatedAddrList []string
+	for _, addr := range addrList {
+		formatedAddrList = append(formatedAddrList, formatAddr(addr))
 	}
+	return strings.Join(formatedAddrList, ", ")
 }
 
 func replaceAtIndex(in string, r rune, i int) string {
diff --git a/pkg/app/popular/utils.go b/pkg/app/popular/utils.go
index c5772d7..bc2a12f 100644
--- a/pkg/app/popular/utils.go
+++ b/pkg/app/popular/utils.go
@@ -16,7 +16,7 @@ func renderPopularThreads(w http.ResponseWriter, templateData interface{}) {
 				Funcs(template.FuncMap{
 					"makeMessage": func(headers map[string][]string) models.Message {
 						return models.Message{
-							Headers: headers,
+							//Headers: headers,
 						}
 					},
 				}).
@@ -28,17 +28,16 @@ func renderPopularThreads(w http.ResponseWriter, templateData interface{}) {
 
 // utility methods
 
-func GetPopularThreads(n int, date string) (models.Threads, error) {
-	var popularThreads models.Threads
-	err := database.DBCon.Model(&popularThreads).
-		TableExpr(`(SELECT id, headers, regexp_replace(regexp_replace(regexp_replace(regexp_replace(headers::jsonb->>'Subject','^\["',''),'"\]$',''),'^Re:\s',''), '^\[.*\]', '') AS c FROM messages WHERE date >= '2020-06-12'::date) t`).
-		ColumnExpr(`c as Subject, jsonb_agg(id)->>0 as Id, jsonb_agg(headers)->>0 as Headers, Count(*) as Count`).
-		GroupExpr(`c`).
-		OrderExpr(`count DESC`).
+func GetPopularThreads(n int, date string) ([]*models.Message, error) {
+
+	var recentMessages []*models.Message
+
+	err := database.DBCon.Model(&recentMessages).
+		OrderExpr("date DESC").
 		Limit(n).
 		Select()
 
-	return popularThreads, err
+	return recentMessages, err
 }
 
 func GetMessagesFromPopularThreads(threads models.Threads) []*models.Message {
diff --git a/pkg/app/search/search.go b/pkg/app/search/search.go
index f6498e9..072bf2f 100644
--- a/pkg/app/search/search.go
+++ b/pkg/app/search/search.go
@@ -44,10 +44,10 @@ func Search(w http.ResponseWriter, r *http.Request) {
 	//
 	var searchResults []*models.Message
 	query := database.DBCon.Model(&searchResults).
-		WhereOr(`headers::jsonb->>'From' LIKE ?`, "%"+searchTerm+"%").
+		WhereOr(`message.from LIKE ?`, "%"+searchTerm+"%").
 		Order("date DESC")
 	if showThreads {
-		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+		query = query.Where(`starts_thread = TRUE`)
 	}
 
 	messagesCount, _ := query.Count()
@@ -65,7 +65,7 @@ func Search(w http.ResponseWriter, r *http.Request) {
 	query = database.DBCon.Model(&searchResults).
 		Where(`tsv_subject @@ to_tsquery(''?'')`, searchTerm)
 	if showThreads {
-		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+		query = query.Where(`starts_thread = TRUE`)
 	}
 
 	messagesCount, _ = query.Count()
@@ -83,7 +83,7 @@ func Search(w http.ResponseWriter, r *http.Request) {
 	query = database.DBCon.Model(&searchResults).
 		Where(`tsv_body @@ to_tsquery(''?'')`, searchTerm)
 	if showThreads {
-		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+		query = query.Where(`starts_thread = TRUE`)
 	}
 
 	messagesCount, _ = query.Count()
diff --git a/pkg/database/connection.go b/pkg/database/connection.go
index 18fa2e6..06514df 100644
--- a/pkg/database/connection.go
+++ b/pkg/database/connection.go
@@ -21,9 +21,16 @@ var (
 func CreateSchema() error {
 	if !tableExists("messages") {
 
-		err := DBCon.CreateTable((*models.Message)(nil), &orm.CreateTableOptions{
-			IfNotExists: true,
-		})
+		for _, model := range []interface{}{(*models.Message)(nil),
+			(*models.MessageToReferences)(nil)} {
+
+			err := DBCon.CreateTable(model, &orm.CreateTableOptions{
+				IfNotExists: true,
+			})
+			if err != nil {
+				return err
+			}
+		}
 
 		// Add tsvector column for subjects
 		DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_subject tsvector;")
@@ -33,7 +40,7 @@ func CreateSchema() error {
 		DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_body tsvector;")
 		DBCon.Exec("CREATE INDEX body_idx ON messages USING gin(tsv_body);")
 
-		return err
+		return nil
 	}
 	return nil
 }
diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go
index 379332c..cdb278d 100644
--- a/pkg/importer/importer.go
+++ b/pkg/importer/importer.go
@@ -2,25 +2,14 @@ package importer
 
 import (
 	"archives/pkg/config"
-	"log"
-	"os"
+	"fmt"
 	"path/filepath"
 )
 
 func FullImport() {
-	err := filepath.Walk(config.MailDirPath(),
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 {
-				if isPublicList(path) {
-					importMail(info.Name(), path, config.MailDirPath())
-				}
-			}
-			return nil
-		})
-	if err != nil {
-		log.Println(err)
-	}
+	fmt.Println("Init import...")
+	filepath.Walk(config.MailDirPath(), initImport)
+	fmt.Println("Start import...")
+	filepath.Walk(config.MailDirPath(), importMail)
+	fmt.Println("Finished import.")
 }
diff --git a/pkg/importer/utils.go b/pkg/importer/utils.go
index b2a5b63..8383ad0 100644
--- a/pkg/importer/utils.go
+++ b/pkg/importer/utils.go
@@ -15,74 +15,217 @@ import (
 	"time"
 )
 
-func importMail(name, path, maildirPath string) {
-	file, _ := os.Open(path)
-	m, _ := mail.ReadMessage(file)
-
-	msg := models.Message{
-		Id:          m.Header.Get("X-Archives-Hash"),
-		Filename:    name,
-		Headers:     m.Header,
-		Attachments: nil,
-		Body:        getBody(m.Header, m.Body),
-		Date:        getDate(m.Header),
-		Lists:       getLists(m.Header),
-		List:        getListName(path),
-		Comment:     "",
-		Hidden:      false,
+type MailIdentifier struct {
+	ArchivesHash string
+	MessageId string
+	To string
+}
+
+// TODO
+var mails []*models.Message
+
+// TODO
+func initImport(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		return err
 	}
+	if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) {
 
-	err := insertMessage(msg)
+		file, _ := os.Open(path)
+		m, _ := mail.ReadMessage(file)
 
+		mails = append(mails, &models.Message{
+			Id:           m.Header.Get("X-Archives-Hash"),
+			Filename:     info.Name(),
+			From:         m.Header.Get("From"),
+			To:           strings.Split(m.Header.Get("To"), ","),
+			Subject:      m.Header.Get("Subject"),
+			MessageId:    m.Header.Get("Message-Id"),
+		})
+	}
+	return nil
+}
+
+// TODO
+func importMail(path string, info os.FileInfo, err error) error {
 	if err != nil {
-		fmt.Println("Error during importing Mail")
-		fmt.Println(err)
+		return err
+	}
+	if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 && isPublicList(path) {
+		file, _ := os.Open(path)
+		m, _ := mail.ReadMessage(file)
+
+		msg := models.Message{
+			Id:          m.Header.Get("X-Archives-Hash"),
+			MessageId:   m.Header.Get("Message-Id"),
+			Filename:    info.Name(),
+			From:        m.Header.Get("From"),
+			To:          strings.Split(m.Header.Get("To"), ","),
+			Cc:          strings.Split(m.Header.Get("Cc"), ","),
+			Subject:     m.Header.Get("Subject"),
+
+			List:        getListName(path),
+
+			// TODO
+			Date:        getDate(m.Header),
+			InReplyToId:   getInReplyToMail(m.Header.Get("In-Reply-To"), m.Header.Get("From")),
+			//References:  getReferencesToMail(strings.Split(m.Header.Get("References"), ","), m.Header.Get("From")),
+			Body:        getBody(m.Header, m.Body),
+			Attachments: getAttachments(m.Header, m.Body),
+
+			StartsThread: m.Header.Get("In-Reply-To") == "" && m.Header.Get("References") == "",
+
+			Comment:     "",
+			Hidden:      false,
+		}
+
+		err := insertMessage(msg)
+
+		if err != nil {
+			fmt.Println("Error during importing Mail")
+			fmt.Println(err)
+		}
+
+		insertReferencesToMail(strings.Split(m.Header.Get("References"), ","), m.Header.Get("X-Archives-Hash"), m.Header.Get("From"))
+
+	}
+	return nil
+}
+
+func getInReplyToMail(messageId, from string) string {
+	// step 1 TODO add description
+	for _, mail := range mails {
+		if mail.MessageId == messageId && strings.Contains(strings.Join(mail.To, ", "), from) {
+			return mail.Id
+		}
+	}
+	// step 2 TODO add description
+	for _, mail := range mails {
+		if mail.MessageId == messageId {
+			return mail.Id
+		}
+	}
+	return ""
+}
+
+
+func insertReferencesToMail(references []string, messageId, from string) []*models.Message {
+	var referencesToMail []*models.Message
+	for _, reference := range references {
+		// step 1 TODO add description
+		for _, mail := range mails {
+			if mail.MessageId == reference  && strings.Contains(strings.Join(mail.To, ", "), from) {
+				referencesToMail = append(referencesToMail, mail)
+			}
+		}
+		// step 2 TODO add description
+		for _, mail := range mails {
+			if mail.MessageId == reference {
+				referencesToMail = append(referencesToMail, mail)
+			}
+		}
+	}
+
+	for _, reference := range referencesToMail {
+		_, err := database.DBCon.Model(&models.MessageToReferences{
+			MessageId: messageId,
+			ReferenceId: reference.Id,
+		}).Insert()
+
+		if err != nil {
+			fmt.Println("Err inserting Message to references")
+			fmt.Println(err)
+		}
 	}
+
+	return referencesToMail
 }
 
 func getDepth(path, maildirPath string) int {
 	return strings.Count(strings.ReplaceAll(path, maildirPath, ""), "/")
 }
 
-func getBody(header mail.Header, body io.Reader) map[string]string {
+func getBody(header mail.Header, body io.Reader) string {
 	if isMultipartMail(header) {
 		boundary := regexp.MustCompile(`boundary="(.*?)"`).
 			FindStringSubmatch(
 				header.Get("Content-Type"))
 		if len(boundary) != 2 {
 			//err
-			return map[string]string{
-				"text/plain": "",
+			return ""
+		}
+		parsedBody := ""
+		mr := multipart.NewReader(body, boundary[1])
+		for {
+			p, err := mr.NextPart()
+			if err != nil {
+				return parsedBody
+			}
+			bodyContent, err := ioutil.ReadAll(p)
+			if err != nil {
+				fmt.Println("Error while reading the body:")
+				fmt.Println(err)
+				continue
+			}
+			if strings.Contains(p.Header.Get("Content-Type"), "text/plain") {
+				return string(bodyContent)
+			} else if strings.Contains(p.Header.Get("Content-Type"), "text/html") {
+				parsedBody = string(bodyContent)
 			}
 		}
-		return getBodyParts(body, boundary[1])
+		return parsedBody
 	} else {
 		content, _ := ioutil.ReadAll(body)
-		return map[string]string{
-			getContentType(header): string(content),
-		}
+		return string(content)
 	}
 }
 
-func getBodyParts(body io.Reader, boundary string) map[string]string {
-	bodyParts := make(map[string]string)
-	mr := multipart.NewReader(body, boundary)
+
+func getAttachments(header mail.Header, body io.Reader) []models.Attachment {
+
+	if !isMultipartMail(header) {
+		return nil
+	}
+
+	boundary := regexp.MustCompile(`boundary="(.*?)"`).
+		FindStringSubmatch(
+			header.Get("Content-Type"))
+	if len(boundary) != 2 {
+		return nil
+	}
+	var attachments []models.Attachment
+	mr := multipart.NewReader(body, boundary[1])
 	for {
 		p, err := mr.NextPart()
 		if err != nil {
-			return bodyParts
+			return attachments
 		}
-		slurp, err := ioutil.ReadAll(p)
+		content, err := ioutil.ReadAll(p)
 		if err != nil {
 			fmt.Println("Error while reading the body:")
 			fmt.Println(err)
 			continue
 		}
-		bodyParts[p.Header.Get("Content-Type")] = string(slurp)
+
+		attachments = append(attachments, models.Attachment{
+			Filename: getAttachmentFileName(p.Header.Get("Content-Type")),
+			Mime:     p.Header.Get("Content-Type"),
+			Content:  string(content),
+		})
+
 	}
-	return bodyParts
+	return attachments
 }
 
+func getAttachmentFileName(contentTypeHeader string) string {
+	parts := strings.Split(contentTypeHeader, "name=")
+	if len(parts) < 2 {
+		return "unknown"
+	}
+	return strings.ReplaceAll(parts[1], "\"", "")
+}
+
+
 func getContentType(header mail.Header) string {
 	contentTypes := regexp.MustCompile(`(.*?);`).
 		FindStringSubmatch(
@@ -103,20 +246,6 @@ func isMultipartMail(header mail.Header) bool {
 	return strings.Contains(getContentType(header), "multipart")
 }
 
-func getLists(header mail.Header) []string {
-	var lists []string
-	// To
-	adr, _ := mail.ParseAddressList(header.Get("To"))
-	for _, v := range adr {
-		lists = append(lists, v.Address)
-	}
-	// Cc
-	adr, _ = mail.ParseAddressList(header.Get("Cc"))
-	for _, v := range adr {
-		lists = append(lists, v.Address)
-	}
-	return lists
-}
 
 func getListName(path string) string {
 	listName := strings.ReplaceAll(path, config.MailDirPath() + ".", "")
@@ -126,8 +255,8 @@ func getListName(path string) string {
 
 func insertMessage(message models.Message) error {
 	_, err := database.DBCon.Model(&message).
-		Value("tsv_subject", "to_tsvector(?)", message.GetSubject()).
-		Value("tsv_body", "to_tsvector(?)", message.GetBody()).
+		Value("tsv_subject", "to_tsvector(?)", message.Subject).
+		Value("tsv_body", "to_tsvector(?)", message.Body).
 		OnConflict("(id) DO NOTHING").
 		Insert()
 	return err
diff --git a/pkg/models/message.go b/pkg/models/message.go
index 6d8b299..89d7d73 100644
--- a/pkg/models/message.go
+++ b/pkg/models/message.go
@@ -1,7 +1,6 @@
 package models
 
 import (
-	"mime"
 	"net/mail"
 	"strings"
 	"time"
@@ -9,23 +8,35 @@ import (
 
 type Message struct {
 	Id       string `pg:",pk"`
+	MessageId string
 	Filename string
 
-	Headers     map[string][]string
-	Body        map[string]string
-	Attachments []Attachment
-
-	Lists []string
 	List string
-	Date  time.Time
 
-	//Search           types.ValueAppender // tsvector
+	From string
+	To []string
+	Cc []string
+
+	Subject string
+	Body string
+
+	Date time.Time
+
+	// fk
+	InReplyTo *Message `pg:"fk:in_reply_to_id"` // fk specifies foreign key
+	InReplyToId string
+
+	// many to many
+	//References []string
+	References []Message `pg:"many2many:message_to_references,joinFK:reference_id"`
+
+	Attachments []Attachment
+
+	StartsThread bool
 
 	Comment string
 	Hidden  bool
 
-	//ParentId         string
-	//Parent           Message -> pg fk?
 }
 
 type Header struct {
@@ -44,12 +55,13 @@ type Attachment struct {
 	Content  string
 }
 
-func (m Message) GetSubject() string {
-	return m.GetHeaderField("Subject")
+type MessageToReferences struct {
+	MessageId string
+	ReferenceId  string
 }
 
 func (m Message) GetListNameFromSubject() string {
-	subject := m.GetSubject()
+	subject := m.Subject
 	listName := strings.Split(subject, "]")[0]
 	listName = strings.ReplaceAll(listName, "[", "")
 	listName = strings.ReplaceAll(listName, "Re:", "")
@@ -58,7 +70,7 @@ func (m Message) GetListNameFromSubject() string {
 }
 
 func (m Message) GetAuthorName() string {
-	addr, err := mail.ParseAddress(m.GetHeaderField("From"))
+	addr, err := mail.ParseAddress(m.From)
 	if err != nil {
 		return ""
 	}
@@ -66,7 +78,7 @@ func (m Message) GetAuthorName() string {
 }
 
 func (m Message) GetMessageId() string {
-	messageId := m.GetHeaderField("Message-Id")
+	messageId := m.MessageId
 	messageId = strings.ReplaceAll(messageId, "<", "")
 	messageId = strings.ReplaceAll(messageId, ">", "")
 	messageId = strings.ReplaceAll(messageId, "\"", "")
@@ -74,83 +86,11 @@ func (m Message) GetMessageId() string {
 }
 
 func (m Message) GetInReplyTo() string {
-	inReplyTo := m.GetHeaderField("In-Reply-To")
+	inReplyTo := m.InReplyTo.MessageId
 	inReplyTo = strings.ReplaceAll(inReplyTo, "<", "")
 	inReplyTo = strings.ReplaceAll(inReplyTo, ">", "")
 	inReplyTo = strings.ReplaceAll(inReplyTo, " ", "")
 	return inReplyTo
 }
 
-func (m Message) GetHeaderField(key string) string {
-	subject, found := m.Headers[key]
-	if !found {
-		return ""
-	}
-	header := strings.Join(subject, " ")
-	if strings.Contains(header, "=?") {
-		dec := new(mime.WordDecoder)
-		decodedHeader, err := dec.DecodeHeader(header)
-		if err != nil {
-			return ""
-		}
-		return decodedHeader
-	}
-	return header
-}
-
-func (m Message) HasHeaderField(key string) bool {
-	_, found := m.Headers[key]
-	return found
-}
-
-func (m Message) GetBody() string {
-	// Get text/plain body
-	for contentType, content := range m.Body {
-		if strings.Contains(contentType, "text/plain") {
-			return content
-		}
-	}
-
-	// If text/plain is not present, fall back to html
-	for contentType, content := range m.Body {
-		if strings.Contains(contentType, "text/html") {
-			return content
-		}
-	}
-
-	// If neither text/plain nor text/html is available return nothing
-	return ""
-}
 
-func (m Message) HasAttachments() bool {
-	for key, _ := range m.Body {
-		if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
-			return true
-		}
-	}
-	return false
-}
-
-func (m Message) GetAttachments() []Attachment {
-	var attachments []Attachment
-	for key, content := range m.Body {
-		if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
-			attachments = append(attachments, Attachment{
-				Filename: getAttachmentFileName(key),
-				Mime:     strings.Split(key, ";")[0],
-				Content:  content,
-			})
-		}
-	}
-	return attachments
-}
-
-// utility methods
-
-func getAttachmentFileName(contentTypeHeader string) string {
-	parts := strings.Split(contentTypeHeader, "name=")
-	if len(parts) < 2 {
-		return "unknown"
-	}
-	return strings.ReplaceAll(parts[1], "\"", "")
-}