Clean up temporary files on write errors, and ignore any temporary service files on load with a warning. This fixes #3207

CHANGELOG.md

@@ -5,6 +5,7 @@ FEATURES:
 IMPROVEMENTS:
 
 BUG FIXES:
+* agent: Clean up temporary files during disk write errors when persisting services and checks. [GH-3207]
 
 ## 0.9.0 (July 20, 2017)
 

agent/agent.go

@@ -1428,7 +1428,7 @@ func (a *Agent) persistService(service *structs.NodeService) error {
 		return err
 	}
 
-	return writeFileAtomic(svcPath, encoded)
+	return writeFileAtomic(svcPath, encoded, a.logger)
 }
 
 // purgeService removes a persisted service definition file from the data dir
@@ -1456,7 +1456,7 @@ func (a *Agent) persistCheck(check *structs.HealthCheck, chkType *structs.CheckT
 		return err
 	}
 
-	return writeFileAtomic(checkPath, encoded)
+	return writeFileAtomic(checkPath, encoded, a.logger)
 }
 
 // purgeCheck removes a persisted check definition file from the data dir
@@ -1470,7 +1470,7 @@ func (a *Agent) purgeCheck(checkID types.CheckID) error {
 
 // writeFileAtomic writes the given contents to a temporary file in the same
 // directory, does an fsync and then renames the file to its real path
-func writeFileAtomic(path string, contents []byte) error {
+func writeFileAtomic(path string, contents []byte, logger *log.Logger) error {
 	uuid, err := uuid.GenerateUUID()
 	if err != nil {
 		return err
@@ -1485,15 +1485,30 @@ func writeFileAtomic(path string, contents []byte) error {
 		return err
 	}
 	if _, err := fh.Write(contents); err != nil {
+		logger.Printf("[INFO] Writing to temp file at %v failed, deleting...\n", tempPath)
+		fh.Close()
+		os.Remove(tempPath)
 		return err
 	}
 	if err := fh.Sync(); err != nil {
+		logger.Printf("[INFO] Syncing temp file at %v failed, deleting...\n", tempPath)
+		fh.Close()
+		os.Remove(tempPath)
 		return err
 	}
 	if err := fh.Close(); err != nil {
+		logger.Printf("[INFO] Closing file handle to temp file at %v failed, deleting...\n", tempPath)
+		fh.Close()
+		os.Remove(tempPath)
 		return err
 	}
-	return os.Rename(tempPath, path)
+	if err := os.Rename(tempPath, path); err != nil {
+		logger.Printf("[INFO] Renaming temp file at %v failed, deleting...\n", tempPath)
+		fh.Close()
+		os.Remove(tempPath)
+		return err
+	}
+	return nil
 }
 
 // AddService is used to add a service entry.
@@ -2072,6 +2087,11 @@ func (a *Agent) loadServices(conf *Config) error {
 			continue
 		}
 
+		// Skip all partially written temporary files
+		if strings.HasSuffix(fi.Name(), "tmp") {
+			a.logger.Printf("[WARN] Ignoring temporary service file %v", fi.Name())
+			continue
+		}
 		// Open the file for reading
 		file := filepath.Join(svcDir, fi.Name())
 		fh, err := os.Open(file)