From 6098086a998f71d1e71c322a46373196dc885a7a Mon Sep 17 00:00:00 2001 From: KHobbits Date: Thu, 17 Apr 2014 04:53:02 +0100 Subject: Optimize uuidmap writing, and use userConf write buffering. --- .../src/com/earth2me/essentials/Essentials.java | 1 + .../com/earth2me/essentials/EssentialsUpgrade.java | 26 ++-- .../src/com/earth2me/essentials/UUIDMap.java | 168 +++++++++++++++++++++ .../src/com/earth2me/essentials/UserMap.java | 99 +++--------- 4 files changed, 208 insertions(+), 86 deletions(-) create mode 100644 Essentials/src/com/earth2me/essentials/UUIDMap.java diff --git a/Essentials/src/com/earth2me/essentials/Essentials.java b/Essentials/src/com/earth2me/essentials/Essentials.java index 35e876841..466c62a3a 100644 --- a/Essentials/src/com/earth2me/essentials/Essentials.java +++ b/Essentials/src/com/earth2me/essentials/Essentials.java @@ -324,6 +324,7 @@ public class Essentials extends JavaPlugin implements net.ess3.api.IEssentials } Economy.setEss(null); Trade.closeLog(); + getUserMap().getUUIDMap().forceWriteUUIDMap(); } @Override diff --git a/Essentials/src/com/earth2me/essentials/EssentialsUpgrade.java b/Essentials/src/com/earth2me/essentials/EssentialsUpgrade.java index d701f0459..d14cdd676 100644 --- a/Essentials/src/com/earth2me/essentials/EssentialsUpgrade.java +++ b/Essentials/src/com/earth2me/essentials/EssentialsUpgrade.java @@ -499,7 +499,7 @@ public class EssentialsUpgrade { return; } - + uuidFileConvert(ess); doneFile.setProperty("uuidFileChange", true); @@ -518,20 +518,23 @@ public class EssentialsUpgrade int countFiles = 0; int countFails = 0; - + int countEssCache = 0; + int countBukkit = 0; + ess.getLogger().info("Found " + userdir.list().length + " files to convert..."); - + for (String string : userdir.list()) { if (!string.endsWith(".yml")) { continue; } - - final int showProgress = countFiles % 1000; - + + final int showProgress = countFiles % 250; + if (showProgress == 0) { + ess.getUserMap().getUUIDMap().forceWriteUUIDMap(); ess.getLogger().info("Converted " + countFiles + "/" + userdir.list().length); } @@ -550,7 +553,7 @@ public class EssentialsUpgrade EssentialsConf conf = new EssentialsConf(file); conf.load(); conf.setProperty("lastAccountName", name); - conf.forceSave(); + conf.save(); String uuidString = conf.getString("uuid", null); @@ -559,6 +562,7 @@ public class EssentialsUpgrade try { uuid = UUID.fromString(uuidString); + countEssCache++; break; } catch (Exception ex2) @@ -569,6 +573,7 @@ public class EssentialsUpgrade if (uuid != null) { + countBukkit++; break; } @@ -582,6 +587,7 @@ public class EssentialsUpgrade if (uuid != null) { + conf.forceSave(); config = new EssentialsUserConf(name, uuid, new File(userdir, uuid + ".yml")); config.convertLegacyFile(); ess.getUserMap().trackUUID(uuid, name); @@ -590,11 +596,11 @@ public class EssentialsUpgrade countFails++; } } + ess.getUserMap().getUUIDMap().forceWriteUUIDMap(); - ess.getLogger().info("Completed Essentials UUID userdata conversion."); - ess.getLogger().info("Attempted to convert " + countFiles + " users. Failed to convert: " + countFails); + ess.getLogger().info("Completed Essentials UUID userdata conversion. Attempted to convert " + countFiles + " users."); + ess.getLogger().info("Converted via cache: " + countEssCache + " :: Converted via lookup: " + countBukkit + " :: Failed to convert: " + countFails); ess.getLogger().info("To rerun the conversion type /essentials uuidconvert"); - } public void beforeSettings() diff --git a/Essentials/src/com/earth2me/essentials/UUIDMap.java b/Essentials/src/com/earth2me/essentials/UUIDMap.java new file mode 100644 index 000000000..fc4a5f0ad --- /dev/null +++ b/Essentials/src/com/earth2me/essentials/UUIDMap.java @@ -0,0 +1,168 @@ +package com.earth2me.essentials; + +import com.google.common.io.Files; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import org.bukkit.Bukkit; + + +public class UUIDMap +{ + private final transient net.ess3.api.IEssentials ess; + private File userList; + private final transient Pattern splitPattern = Pattern.compile(","); + private static final ExecutorService EXECUTOR_SERVICE = Executors.newSingleThreadExecutor(); + private final AtomicInteger pendingDiskWrites = new AtomicInteger(0); + + public UUIDMap(final net.ess3.api.IEssentials ess) + { + this.ess = ess; + userList = new File(ess.getDataFolder(), "usermap.csv"); + + } + + public void loadAllUsers(final ConcurrentSkipListMap names) + { + + try + { + if (!userList.exists()) + { + userList.createNewFile(); + } + + final BufferedReader reader = new BufferedReader(new FileReader(userList)); + try + { + do + { + final String line = reader.readLine(); + if (line == null) + { + break; + } + else + { + String[] values = splitPattern.split(line); + if (values.length == 2) + { + names.put(values[0], UUID.fromString(values[1])); + } + } + } + while (true); + } + finally + { + reader.close(); + } + } + catch (IOException ex) + { + Bukkit.getLogger().log(Level.SEVERE, ex.getMessage(), ex); + } + + } + + public void writeUUIDMap() + { + _writeUUIDMap(); + } + + public void forceWriteUUIDMap() + { + try + { + Future future = _writeUUIDMap();; + if (future != null) + { + future.get(); + } + } + catch (InterruptedException ex) + { + ess.getLogger().log(Level.SEVERE, ex.getMessage(), ex); + } + catch (ExecutionException ex) + { + ess.getLogger().log(Level.SEVERE, ex.getMessage(), ex); + } + } + + public Future _writeUUIDMap() + { + final ConcurrentSkipListMap names = ess.getUserMap().getNames().clone(); + ess.getLogger().info("I see " + names.size() + " in my name map!"); + pendingDiskWrites.incrementAndGet(); + Future future = EXECUTOR_SERVICE.submit(new WriteRunner(ess.getDataFolder(), userList, names, pendingDiskWrites)); + return future; + } + + + private static class WriteRunner implements Runnable + { + private final File location; + private final File endFile; + private final ConcurrentSkipListMap names; + private final AtomicInteger pendingDiskWrites; + + private WriteRunner(final File location, final File endFile, final ConcurrentSkipListMap names, final AtomicInteger pendingDiskWrites) + { + this.location = location; + this.endFile = endFile; + this.names = names; + this.pendingDiskWrites = pendingDiskWrites; + } + + @Override + public void run() + { + synchronized (location) + { + if (pendingDiskWrites.get() > 1) + { + pendingDiskWrites.decrementAndGet(); + return; + } + + try + { + File configFile = File.createTempFile("usermap", ".tmp.yml", location); + + final BufferedWriter bWriter = new BufferedWriter(new FileWriter(configFile)); + for (Map.Entry entry : names.entrySet()) + { + bWriter.write(entry.getKey() + "," + entry.getValue().toString()); + bWriter.newLine(); + } + + bWriter.close(); + Files.move(configFile, endFile); + } + catch (IOException ex) + { + Logger.getLogger(UserMap.class.getName()).log(Level.SEVERE, null, ex); + } + finally + { + pendingDiskWrites.decrementAndGet(); + } + } + } + } +} \ No newline at end of file diff --git a/Essentials/src/com/earth2me/essentials/UserMap.java b/Essentials/src/com/earth2me/essentials/UserMap.java index 951c92125..ee34491a2 100644 --- a/Essentials/src/com/earth2me/essentials/UserMap.java +++ b/Essentials/src/com/earth2me/essentials/UserMap.java @@ -4,26 +4,15 @@ import com.earth2me.essentials.utils.StringUtil; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; -import com.google.common.io.Files; import com.google.common.util.concurrent.UncheckedExecutionException; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; import java.util.Collections; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ExecutionException; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; import net.ess3.api.IEssentials; -import org.bukkit.Bukkit; import org.bukkit.entity.Player; @@ -33,14 +22,13 @@ public class UserMap extends CacheLoader implements IConf private final transient Cache users; private final transient ConcurrentSkipListSet keys = new ConcurrentSkipListSet(); private final transient ConcurrentSkipListMap names = new ConcurrentSkipListMap(); - private final transient Pattern splitPattern = Pattern.compile(","); - private File userList; + private UUIDMap uuidMap; public UserMap(final IEssentials ess) { super(); this.ess = ess; - userList = new File(ess.getDataFolder(), "usermap.csv"); + uuidMap = new UUIDMap(ess); users = CacheBuilder.newBuilder().maximumSize(ess.getSettings().getMaxUserCacheCount()).softValues().build(this); loadAllUsersAsync(ess); } @@ -77,43 +65,7 @@ public class UserMap extends CacheLoader implements IConf } } - try - { - if (!userList.exists()) - { - userList.createNewFile(); - } - - final BufferedReader reader = new BufferedReader(new FileReader(userList)); - try - { - do - { - final String line = reader.readLine(); - if (line == null) - { - break; - } - else - { - String[] values = splitPattern.split(line); - if (values.length == 2) - { - names.put(values[0], UUID.fromString(values[1])); - } - } - } - while (true); - } - finally - { - reader.close(); - } - } - catch (IOException ex) - { - Bukkit.getLogger().log(Level.SEVERE, ex.getMessage(), ex); - } + uuidMap.loadAllUsers(names); } }); @@ -185,31 +137,12 @@ public class UserMap extends CacheLoader implements IConf { if (uuid != null) { - names.put(StringUtil.sanitizeFileName(name), uuid); keys.add(uuid); - writeUUIDMap(); - } - } - - public void writeUUIDMap() - { - try - { - final File tempFile = File.createTempFile("usermap", ".tmp.yml", ess.getDataFolder()); - final BufferedWriter bWriter = new BufferedWriter(new FileWriter(tempFile)); - - for (Map.Entry entry : names.entrySet()) + if (name != null && name.length() > 0) { - bWriter.write(entry.getKey() + "," + entry.getValue().toString()); - bWriter.newLine(); + names.put(StringUtil.sanitizeFileName(name), uuid); + uuidMap.writeUUIDMap(); } - - bWriter.close(); - Files.move(tempFile, userList); - } - catch (IOException ex) - { - Logger.getLogger(UserMap.class.getName()).log(Level.SEVERE, null, ex); } } @@ -219,15 +152,18 @@ public class UserMap extends CacheLoader implements IConf Player player = ess.getServer().getPlayer(uuid); if (player != null) { - return new User(player, ess); + final User user = new User(player, ess); + trackUUID(uuid, user.getName()); + return user; } final File userFile = getUserFileFromID(uuid); if (userFile.exists()) { - keys.add(uuid); - return new User(new OfflinePlayer(uuid, ess.getServer()), ess); + final User user = new User(new OfflinePlayer(uuid, ess.getServer()), ess); + trackUUID(uuid, user.getName()); + return user; } throw new Exception("User not found!"); @@ -236,6 +172,7 @@ public class UserMap extends CacheLoader implements IConf @Override public void reloadConfig() { + getUUIDMap().forceWriteUUIDMap(); loadAllUsersAsync(ess); } @@ -261,6 +198,16 @@ public class UserMap extends CacheLoader implements IConf return keys.size(); } + public ConcurrentSkipListMap getNames() + { + return names; + } + + public UUIDMap getUUIDMap() + { + return uuidMap; + } + private File getUserFileFromID(final UUID uuid) { final File userFolder = new File(ess.getDataFolder(), "userdata"); -- cgit v1.2.3