Web cache architecture and cache design

  • View
    123

  • Download
    11

Embed Size (px)

DESCRIPTION

Web cache architecture and cache design

Transcript

  • kim
  • What is ?
  • file cache shared cache local cache client cache CDN cdn cache etc
  • Client Cache
  • LRU libevent O(1)
  • Why use
  • iNode
  • /queue/all
  • /dev/shm swap
  • /dev/shm/secrectkey
  • pfs /dev/shm/pfs_domain
  • PHP APC http://php.net/apc/ apc_compiler_cache apc_user_cache
  • -- Nginx : lua_shared_dict -- local badwords local lcache = ngx.shared['lcache'] if lcache then badwords = lcache:get(KEY_BADWORDS) end if not badwords then local hc = self:_getHttpObj() badwords = hc:get('api.xxx.com', '/API/black_word') if badwords then lcache:set(KEY_BADWORDS, badwords, 600) end end
  • Socket
  • Memcached
  • 90%
  • // $key = $account . '@reg_' . $regType; $userExist = $memd->Get($key); if (!$userExist) { $userLoginX = new UserLoginX(); $userExist = $userLoginX->checkExists($account); if ($userExist) { $memd->Put($key, "1", 10800); } }
  • // $cacheKey = md5('api@db_promo@2012' . $num); $videoList = $memd->Get($cacheKey); if (!$videoList) { $data_json = Http::Get("api.xxx.com", API/api_bd_list); if ($data_json) { $videoList = json_decode($data_json, true); $memd->Put($cacheKey, $videoList, 1800); } }
  • // $data = array ("startTime" => time (), "num" => $msgNum ); $cache_key = "memKeyPreLimit" . $userid; $rs = $memd->Get ($cache_key); if (isset($rs) && is_array ( $rs )) { $startTime = $rs ["startTime"]; $num = $rs ["num"]; if ((time () - $startTime) < 900 && ($num + $msgNum < 30)) { $data = array ("startTime" => $startTime, "num" => $num + $msgNum ); } } $memd->Put($cache_key, json_encode( $data ), 900);
  • session
  • // $cacheKey = 'key_list_fresh'; $backupKey = 'key_list_backup'; $videoList = $memd->Get($cacheKey); if (!$videoList) { $data_json = Http::Get(api.xxx.com", 'API/list'); if ($data_json) { $videoList = json_decode($data_json, true); $memd->Put($cacheKey, $videoList, 1800); $memd->Put($backupKey, $videoList, 86400); // } else { $videoList = $memd->Get($backupKey); // } }
  • TtServer
  • 5%commentmyfav Memcached DBM
  • // protected function _fetchContent($key) { $result = ''; $localConfig = $GLOBALS['scfg']['tt_server']; if (!$this->_dbContent) { $this->_dbContent = new Mem($localConfig); } $content = $this->_dbContent->Get($key); $result = sprintf('%s', json_decode($content)); if (json_last_error() != JSON_ERROR_NONE) { $result = sprintf('%s', json_decode(json_encode($content))); } return $result; }
  • KtServer
  • 10%
  • redis hash
  • Redis
  • 50%
  • // incrincrbydecrdecrby // 590 $rc = $redis->get($rcKey); if (!$rc) { $redis->setex($rcKey, 1, 86400); } $rs = $redis->get($rKey); if ($rs === false && $rc < 90) { $redis->setex($rKey, 1, 5); $redis->incr($rcKey); } else { throw new Exception(''); }
  • // $total = $redis->get($rKey); if ($total === false) { $total = $obj->getTotal($id); if ($total > 300000) { $res = $redis->set($rKey, $total); } else if ($total > 99999) { $res = $redis->setex($rKey, $total, 86400 * 72); } else if ($total > 9999) { $res = $redis->setex($rKey, $total, 86400 * 6); } else { $res = $redis->setex($rKey, $total, 3600); } }
  • // lpush/ltrim/lpop/lrange $listData = getUserDianBoList($userId); foreach ($listData as $data) { $count = $redis->LPush($uKey, $data); if ($count > MAX_RECORD_NUM_PER_LIST) { $redis->LTrim($uKey, 0, MAX_RECORD_NUM_PER_USER - 1); } } $list = $redis -> lRange($uKey, 0, MAX_RECORD_NUM_PER_USER - 1);
  • -- hmset local rhkey = 'tb|updates' local hData = {} hData[tableIndex] = cjson.encode(currentTime) rds:hmset(rhkey, hData) local tableRecords = rds:hgetall($rhkey); if tableRecords then ....... end
  • -- hmget / hmset -- rds:hclear(hlkey) local list local hlkey = 'hashlist|' .. vid local hlfield = orderby .. '|' .. num local hResult = rds:hmget(hlkey, hlfield) if hResult and hResult[1] then list = cjson.decode(hResult[1]) else list = self:_getList(vid, orderby, num) local hData = {} hData[hlfield] = cjson.encode(list) rds:hmset(hlkey, hData) end
  • // zset // 60 public function checkDuplicatedCnt($vid, $content) { $redis = self::getRedis(); $md5Cnt = md5($content); $rKey = cnt| . $vid; $isDuplicated = !$redis->sAdd($rKey, $md5Cnt); $redis->setTimeoutOnce($rKey, 60); return $isDuplicated; }
  • // zIncrBy zRevRange Top 20 $rKey = 'toplist|' . date('md', time()); $score = 1; $result = $redis->zIncrBy($rKey, $vid, $score); $redis->setTimeoutOnce($rKey, 86400); ...... $result = $redis->zRevRange($rKey, 0, 19);
  • -- local zsetData = {} local rzkey = table|updated table.insert(zsetData, updated_at) -- table.insert(zsetData, tablehash) -- rds:zadd(rzkey, unpack(zsetData)) local withScore = true; local tableRecords = rds:zrevrange(table|updated, 0, 1000, withScore);
  • -- pipeline rds:init_pipeline() local zsetData = {} local hashData = {} for k,v in pairs(tblVinfo) do -- table.insert(zsetData, v['times']) table.insert(zsetData, v['id']) -- hashData[tostring(v['id'])] = cjson.encode(v) end rds:zadd(rzkey, unpack(zsetData)) rds:hmset(rhkey, hashData) -- mset/mget / rds:commit_pipeline()
  • // MULTI // 2 push $pipe = $redis->multi(); foreach ($result['data'] as $value) { $len = $pipe->rPush($rKey, json_encode($value)); } $pipe->setTimeout($rKey, 600); $pipe->exec();
  • SSDB
  • leveldb redis
  • CDN
  • // cdn if ($videoInfo['chk'] == 'n') { //600S $cdnCacheTime = 600; } else { $cdnCacheTime = 259200; } header("Expires: " . gmdate("D, d M Y H:i:s", time() + $cdnCacheTime) . " GMT"); header("Cache-Control: max-age=" . $cdnCacheTime); header("Last-Modified: " . gmdate("D, d M Y H:i:s", time()) . " GMT"); if (ob_get_length()) { header("Content-Length: " . ob_get_length()); } ob_end_flush();
  • twemproxy
  • 70% memcached ascii redis
  • -- -- --
  • local res = db:findOne(sql) if type(res) == 'table' and res['id'] then rds:setex(key, cjson.encode(res), 3600) elseif db.lasterr == db.ERROR_NOT_FOUND then rds:setex(key, cjson.encode(res), 60) else -- end
  • RESTFUL
  • AB
  • 1.
  • 2.
  • 3.