Система кэширования Java (JCS): загрузка 300 МБ данных в кэш занимает 2-3 часа
Сценарий: у меня есть 2-3 java хеш-карты (80 черных записей: около 300 МБ). Я загружаю его в кеш диска JCS. Загрузка занимает 2-3 часа! Вот код для загрузки данных! пример упаковки;
открытый класс Singleton {
private static Singleton single_instance = null;
public JCS cache_bestfeatures;
public JCS cache_tposmap;
public JCS cache_tnegmap;
public JCS cache_tnetmap;
public JCS cache_pos;
public JCS cache_neg;
public JCS cache_net;
public JCS cache_total;
@SuppressWarnings("unchecked")
private Singleton() throws CacheException, FileNotFoundException, IOException, ClassNotFoundException
{
CompositeCacheManager ccm = CompositeCacheManager.getUnconfiguredInstance();
Properties props = new Properties();
props.put("jcs.default", "DC");
props.put("jcs.default.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.default.cacheattributes.MaxObjects", "1");
props.put("jcs.default.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.default.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.default.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.default.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.default.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.default.elementattributes.IsEternal", "true");
props.put("jcs.default.elementattributes.MaxLifeSeconds", "21600");
props.put("jcs.default.elementattributes.IdleTime", "1800");
props.put("jcs.default.elementattributes.IsSpool", "true");
props.put("jcs.default.elementattributes.IsRemote", "true");
props.put("jcs.default.elementattributes.IsLateral", "true");
props.put("jcs.region.bestfeatures", "DC_BestFeatures");
props.put("jcs.region.bestfeatures.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.bestfeatures.cacheattributes.MaxObjects", "1");
props.put("jcs.region.bestfeatures.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.bestfeatures.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.bestfeatures.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.bestfeatures.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.bestfeatures.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.bestfeatures.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.bestfeatures.elementattributes.IsEternal", "true");
props.put("jcs.auxiliary.DC_BestFeatures", "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory");
props.put("jcs.auxiliary.DC_BestFeatures.attributes", "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes");
props.put("jcs.auxiliary.DC_BestFeatures.attributes.DiskPath", "/home/alethe/workspace/example/src/example");
props.put("jcs.auxiliary.DC_BestFeatures.attributes.MaxPurgatorySize", "10000");
props.put("jcs.auxiliary.DC_BestFeatures.attributes.MaxKeySize", "10000000");
props.put("jcs.auxiliary.DC_general", "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory");
props.put("jcs.auxiliary.DC_general.attributes", "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes");
props.put("jcs.auxiliary.DC_general.attributes.DiskPath", "/home/alethe/workspace/example/src/example");
props.put("jcs.auxiliary.DC_general.attributes.MaxPurgatorySize", "10000");
props.put("jcs.auxiliary.DC_general.attributes.MaxKeySize", "10000000");
props.put("jcs.region.tposmap", "DC_general");
props.put("jcs.region.tposmap.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.tposmap.cacheattributes.MaxObjects", "1");
props.put("jcs.region.tposmap.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.tposmap.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.tposmap.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.tposmap.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.tposmap.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.tposmap.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.tposmap.elementattributes.IsEternal", "true");
props.put("jcs.region.tnegmap", "DC_general");
props.put("jcs.region.tnegmap.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.tnegmap.cacheattributes.MaxObjects", "1");
props.put("jcs.region.tnegmap.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.tnegmap.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.tnegmap.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.tnegmap.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.tnegmap.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.tnegmap.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.tnegmap.elementattributes.IsEternal", "true");
props.put("jcs.region.tnetmap", "DC_general");
props.put("jcs.region.tnetmap.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.tnetmap.cacheattributes.MaxObjects", "1");
props.put("jcs.region.tnetmap.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.tnetmap.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.tnetmap.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.tnetmap.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.tnetmap.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.tnetmap.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.pos", "DC_general");
props.put("jcs.region.pos.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.pos.cacheattributes.MaxObjects", "1");
props.put("jcs.region.pos.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.pos.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.pos.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.pos.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.pos.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.pos.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.pos.elementattributes.IsEternal", "true");
props.put("jcs.region.neg", "DC_general");
props.put("jcs.region.neg.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.neg.cacheattributes.MaxObjects", "1");
props.put("jcs.region.neg.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.neg.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.neg.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.neg.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.neg.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.neg.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.neg.elementattributes.IsEternal", "true");
props.put("jcs.region.net", "DC_general");
props.put("jcs.region.net.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.net.cacheattributes.MaxObjects", "1");
props.put("jcs.region.net.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.net.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.net.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.net.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.net.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.net.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.net.elementattributes.IsEternal", "true");
props.put("jcs.region.total", "DC_general");
props.put("jcs.region.total.cacheattributes", "org.apache.jcs.engine.CompositeCacheAttributes");
props.put("jcs.region.total.cacheattributes.MaxObjects", "1");
props.put("jcs.region.total.cacheattributes.MemoryCacheName", "org.apache.jcs.engine.memory.lru.LRUMemoryCache");
props.put("jcs.region.total.cacheattributes.UseMemoryShrinker", "false");
props.put("jcs.region.total.cacheattributes.MaxMemoryIdleTimeSeconds", "3600");
props.put("jcs.region.total.cacheattributes.ShrinkerIntervalSeconds", "60");
props.put("jcs.region.total.cacheattributes.MaxSpoolPerRun", "500");
props.put("jcs.region.total.elementattributes", "org.apache.jcs.engine.ElementAttributes");
props.put("jcs.region.total.elementattributes.IsEternal", "true");
ccm.configure(props);
cache_bestfeatures = JCS.getInstance( "bestfeatures" );
cache_tposmap = JCS.getInstance( "tposmap" );
cache_tnegmap = JCS.getInstance( "tnegmap" );
cache_tnetmap = JCS.getInstance( "tnetmap" );
cache_pos = JCS.getInstance( "pos" );
cache_neg = JCS.getInstance( "neg" );
cache_net = JCS.getInstance( "net" );
cache_total = JCS.getInstance( "total" );
HashSet<Pair<String, String>> bestfeatures_tmp = new HashSet<Pair<String, String>> ();
ConcurrentHashMap<Pair<String, String>, Float> pos_tmp = new ConcurrentHashMap<Pair<String, String>, Float>();
ConcurrentHashMap<Pair<String,String>, Float> neg_tmp = new ConcurrentHashMap<Pair<String,String>, Float>();
ConcurrentHashMap<Pair<String,String>, Float> net_tmp = new ConcurrentHashMap<Pair<String,String>, Float>();
HashMap<String, Float> tposMap_tmp = new HashMap<String, Float>();
HashMap<String, Float> tnegMap_tmp = new HashMap<String, Float>();
HashMap<String, Float> tnetMap_tmp = new HashMap<String, Float>();
HashMap<Pair<String, String>, Float> total_tmp = new HashMap<Pair<String, String>, Float>();
String train_folder_path = "/home/alethe/Desktop/JCS/TrainingParam/";
int j = 0;
System.out.println("Here");
ObjectInputStream object_best_feature = new ObjectInputStream(new FileInputStream( train_folder_path + "best_features.tmp"));
bestfeatures_tmp = (HashSet<Pair<String, String>>) object_best_feature.readObject();
System.out.println("Top features loaded with count : "+ bestfeatures_tmp.size());
Iterator<Pair<String, String>> iterator = bestfeatures_tmp.iterator();
while (iterator.hasNext()) {
Pair<String, String> p = iterator.next();
cache_bestfeatures.put(p, true);
System.out.println(j + " ");
j = j + 1;
iterator.remove();
}
object_best_feature.close();
System.out.println("Here");
ObjectInputStream object_tpos = new ObjectInputStream(new FileInputStream(train_folder_path + "tposMap.tmp"));
tposMap_tmp = (HashMap<String, Float>) object_tpos.readObject();
System.out.println("tposMap : "+ tposMap_tmp.size());
Iterator<Map.Entry<String,Float>> iter = tposMap_tmp.entrySet().iterator();
while (iter.hasNext())
{
Map.Entry<String,Float> entry = iter.next();
cache_tposmap.put(entry.getKey(), entry.getValue());
iter.remove();
}
object_tpos.close();
System.out.println("Here");
ObjectInputStream object_tneg = new ObjectInputStream(new FileInputStream(train_folder_path + "tnegMap.tmp"));
tnegMap_tmp = (HashMap<String, Float>) object_tneg.readObject();
System.out.println("tnegMap : "+ tnegMap_tmp.size());
iter = tnegMap_tmp.entrySet().iterator();
while (iter.hasNext())
{
Map.Entry<String,Float> entry = iter.next();
cache_tnegmap.put(entry.getKey(), entry.getValue());
iter.remove();
}
object_tneg.close();
System.out.println("Here");
ObjectInputStream object_tnet = new ObjectInputStream(new FileInputStream(train_folder_path + "tnetMap.tmp"));
tnetMap_tmp = (HashMap<String, Float>) object_tnet.readObject();
System.out.println("tnetMap : "+ tnetMap_tmp.size());
iter = tnetMap_tmp.entrySet().iterator();
while (iter.hasNext())
{
Map.Entry<String,Float> entry = iter.next();
cache_tnetmap.put(entry.getKey(), entry.getValue());
iter.remove();
}
object_tnet.close();
System.out.println("Here");
ObjectInputStream object_pos = new ObjectInputStream(new FileInputStream(train_folder_path + "pos.tmp"));
pos_tmp = (ConcurrentHashMap<Pair<String, String>, Float>) object_pos.readObject();
System.out.println("pos : "+ pos_tmp.size());
int i = 0;
Iterator<Map.Entry<Pair<String, String>, Float>> iter2 = pos_tmp.entrySet().iterator();
while (iter2.hasNext())
{
Map.Entry<Pair<String, String>, Float> entry = iter2.next();
cache_pos.put(entry.getKey(), entry.getValue());
iter2.remove();
System.out.println(i + " ");
i = i + 1;
}
object_pos.close();
i = 0;
System.out.println("Here");
ObjectInputStream object_neg = new ObjectInputStream(new FileInputStream(train_folder_path + "neg.tmp"));
neg_tmp = (ConcurrentHashMap<Pair<String, String>, Float>) object_neg.readObject();
System.out.println("Neg : "+ neg_tmp.size());
iter2 = neg_tmp.entrySet().iterator();
while (iter2.hasNext())
{
Map.Entry<Pair<String, String>, Float> entry = iter2.next();
cache_neg.put(entry.getKey(), entry.getValue());
iter2.remove();
System.out.println(i + " ");
i = i + 1;
}
object_neg.close();
i = 0;
System.out.println("Here");
ObjectInputStream object_net = new ObjectInputStream(new FileInputStream(train_folder_path + "net.tmp"));
net_tmp = (ConcurrentHashMap<Pair<String, String>, Float>) object_net.readObject();
System.out.println("Net : "+ net_tmp.size());
iter2 = net_tmp.entrySet().iterator();
while (iter2.hasNext())
{
Map.Entry<Pair<String, String>, Float> entry = iter2.next();
cache_net.put(entry.getKey(), entry.getValue());
iter2.remove();
System.out.println(i + " ");
i = i + 1;
}
object_net.close();
int z = 0;
System.out.println("Here");
ObjectInputStream object_total = new ObjectInputStream(new FileInputStream(train_folder_path + "total.tmp"));
total_tmp = (HashMap<Pair<String, String>, Float>) object_total.readObject();
for(Map.Entry<Pair<String, String>, Float> entry : total_tmp.entrySet())
{
cache_total.put(entry.getKey(), entry.getValue());
System.out.println(z + " ");
z = z + 1;
}
System.out.println("total : "+ total_tmp.size());
total_tmp.clear();
object_total.close();
}
public static Singleton getInstance() throws CacheException, FileNotFoundException, ClassNotFoundException, IOException
{
if (single_instance == null)
single_instance = new Singleton();
return single_instance;
}
}
Вопрос: почему это занимает 2-3 часа? Что-то я неправильно настраиваю в конфигурации JCS?