implement /replace in dcs-index-backend

dcs-package-importer triggers loading the new shard (and deleting the
old one afterwards) after merging index files.

cmd/dcs-index-backend/index-backend.go

@@ -10,7 +10,7 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path"
+	"path/filepath"
 	"runtime/pprof"
 	"time"
 )
@@ -72,17 +72,56 @@ func Index(w http.ResponseWriter, r *http.Request) {
 	fmt.Printf("[%s] written in %v\n", id, t3.Sub(t2))
 }
 
+// XXX: theoretically, we need to make sure no Index() call is currently in
+// progress since they may return incorrect result if “ix” suddenly points to a
+// different index.
+func Replace(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	newShard := r.Form.Get("shard")
+
+	file, err := os.Open(filepath.Dir(*indexPath))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+	names, err := file.Readdirnames(-1)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, name := range names {
+		if name == newShard {
+			newShard = filepath.Join(filepath.Dir(*indexPath), name)
+			// We verified the given argument refers to an index shard within
+			// this directory, so let’s load this shard.
+			oldIndex := ix
+			log.Printf("Trying to load %q\n", newShard)
+			ix = index.Open(newShard)
+			// Overwrite the old full shard with the new one. This is necessary
+			// so that the state is persistent across restarts and has the nice
+			// side-effect of cleaning up the old full shard.
+			if err := os.Rename(newShard, *indexPath); err != nil {
+				log.Fatal(err)
+			}
+			oldIndex.Close()
+			return
+		}
+	}
+
+	http.Error(w, "No such shard.", http.StatusInternalServerError)
+}
+
 func main() {
 	flag.Parse()
 	if *indexPath == "" {
 		log.Fatal("You need to specify a non-empty -index_path")
 	}
 	fmt.Println("Debian Code Search index-backend")
 
-	id = path.Base(*indexPath)
-
+	id = filepath.Base(*indexPath)
 	ix = index.Open(*indexPath)
 
 	http.HandleFunc("/index", Index)
+	http.HandleFunc("/replace", Replace)
 	log.Fatal(http.ListenAndServe(*listenAddress, nil))
 }

cmd/dcs-package-importer/importer.go

@@ -186,6 +186,28 @@ func mergeToShard() {
 	//	log.Printf("merged in %v\n", t1.Sub(t0))
 	//}
 	log.Printf("merged into shard %s\n", tmpIndexPath.Name())
+
+	// If full.idx does not exist (i.e. on initial deployment), just move the
+	// new index to full.idx, the dcs-index-backend will not be running anyway.
+	fullIdxPath := filepath.Join(*unpackedPath, "full.idx")
+	if _, err := os.Stat(fullIdxPath); os.IsNotExist(err) {
+		if err := os.Rename(tmpIndexPath.Name(), fullIdxPath); err != nil {
+			log.Fatal(err)
+		}
+		return
+	}
+
+	// Replace the current index with the newly created index.
+	resp, err := http.Get(fmt.Sprintf("http://localhost:28081/replace?shard=%s", filepath.Base(tmpIndexPath.Name())))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		log.Fatalf("dcs-index-backend /replace response: %+v (body: %s)\n", resp, body)
+	}
 }
 
 func indexPackage(pkg string) {

index/mmap_linux.go

@@ -21,11 +21,11 @@ func mmapFile(f *os.File) mmapData {
 	}
 	n := int(size)
 	if n == 0 {
-		return mmapData{f, nil}
+		return mmapData{f, nil, nil}
 	}
 	data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_SHARED)
 	if err != nil {
 		log.Fatalf("mmap %s: %v", f.Name(), err)
 	}
-	return mmapData{f, data[:n]}
+	return mmapData{f, data[:n], data}
 }

index/read.go

@@ -71,6 +71,7 @@ import (
 	"os"
 	"runtime"
 	"sort"
+	"syscall"
 )
 
 const (
@@ -112,6 +113,13 @@ func Open(file string) *Index {
 	return ix
 }
 
+func (ix *Index) Close() {
+	if err := syscall.Munmap(ix.data.orig); err != nil {
+		log.Fatalf("munmap: %v", err)
+	}
+	ix.data.f.Close()
+}
+
 // slice returns the slice of index data starting at the given byte offset.
 // If n >= 0, the slice must have length at least n and is truncated to length n.
 func (ix *Index) slice(off uint32, n int) []byte {
@@ -428,8 +436,9 @@ func corrupt(file string) {
 
 // An mmapData is mmap'ed read-only data from a file.
 type mmapData struct {
-	f *os.File
-	d []byte
+	f    *os.File
+	d    []byte
+	orig []byte
 }
 
 // mmap maps the given file into memory.