Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support incremental configuration synchronization client #5325

Closed
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
code format
jackie-coming committed Nov 27, 2024
commit 8cafda21f885f610438fd38e7e560b4013275d68
Original file line number Diff line number Diff line change
@@ -56,8 +56,6 @@ public class BizConfig extends RefreshableConfig {
private static final int DEFAULT_LONG_POLLING_TIMEOUT = 60; //60s
public static final int DEFAULT_RELEASE_HISTORY_RETENTION_SIZE = -1;

private static final int CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE = 100;

private static final Gson GSON = new Gson();

private static final Type namespaceValueLengthOverrideTypeReference =
@@ -248,10 +246,6 @@ public boolean isConfigServiceChangeCacheEnabled() {
return getBooleanProperty("config-service.change.cache.enabled", false);
}

public int configServiceHistoryCacheHistoryMaxSize() {
int maxSize = getIntProperty("config-service.change.cache.history.maxSize", CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE);
return checkInt(maxSize, 1, Integer.MAX_VALUE, CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE);
}

int checkInt(int value, int min, int max, int defaultValue) {
if (value >= min && value <= max) {
Original file line number Diff line number Diff line change
@@ -118,11 +118,11 @@ public List<Release> findByReleaseIds(Set<Long> releaseIds) {
public List<Release> findByReleaseKeys(Set<String> releaseKeys) {
return releaseRepository.findByReleaseKeyIn(releaseKeys);
}

public Release findByReleaseKey(String releaseKey) {
return releaseRepository.findByReleaseKey(releaseKey);
}


public Release findLatestActiveRelease(Namespace namespace) {
return findLatestActiveRelease(namespace.getAppId(),
namespace.getClusterName(), namespace.getNamespaceName());
Original file line number Diff line number Diff line change
@@ -46,7 +46,12 @@
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

/**
@@ -151,30 +156,40 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String

ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace,
latestMergedReleaseKey);

Map<String, String> latestConfigurations=mergeReleaseConfigurations(releases);
//增量配置开关

if(bizConfig.isConfigServiceChangeCacheEnabled()){
//将clientSideReleaseKey用字符串+ 拆出来,同时按照顺序
LinkedHashSet<String> clientSideReleaseKeys= Sets.newLinkedHashSet(Arrays.stream(clientSideReleaseKey.split("\\+")).collect(Collectors.toList()));
Map<String,Release> historyReleasesMap=configService.findReleasesByReleaseKeys(clientSideReleaseKeys);
if(historyReleasesMap!=null){
//按照顺序merge
List<Release> historyReleases=new ArrayList<>();
for (String clientSideReleaseKeyItem:clientSideReleaseKeys){
Release release=historyReleasesMap.get(clientSideReleaseKeyItem);
LinkedHashSet<String> clientSideReleaseKeys = Sets.newLinkedHashSet(
Arrays.stream(clientSideReleaseKey.split("\\+")).collect(Collectors.toList()));

Map<String, Release> historyReleases = configService.findReleasesByReleaseKeys(
clientSideReleaseKeys);
//find history releases
if (historyReleases != null) {
//order by clientSideReleaseKeys
List<Release> historyReleasesWithOrder = new ArrayList<>();
for (String item : clientSideReleaseKeys) {
Release release = historyReleases.get(item);
if(release!=null){
historyReleases.add(release);
historyReleasesWithOrder.add(release);
}
}
Map<String, String> historyConfigurations=mergeReleaseConfigurations(historyReleases);
List<ConfigurationChange> configurationChanges=configService.calcConfigurationChanges(latestConfigurations, historyConfigurations);

Map<String, String> historyConfigurations = mergeReleaseConfigurations
(historyReleasesWithOrder);

List<ConfigurationChange> configurationChanges = configService.calcConfigurationChanges
(latestConfigurations, historyConfigurations);

apolloConfig.setConfigurationChanges(configurationChanges);

apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTALSYNC.getValue());
return apolloConfig;

}

}
//change计算历史和最新的配置

apolloConfig.setConfigurations(latestConfigurations);

Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded,

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -21,9 +21,17 @@
import com.ctrip.framework.apollo.core.ConfigConsts;
import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages;

import com.ctrip.framework.apollo.core.dto.ConfigurationChange;
import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType;
import com.google.common.base.Strings;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

/**
* @author Jason Song(song_s@ctrip.com)
@@ -93,6 +101,52 @@ private Release findRelease(String clientAppId, String clientIp, String clientLa
return release;
}

public List<ConfigurationChange> calcConfigurationChanges(
Map<String, String> latestReleaseConfigurations, Map<String, String> historyConfigurations) {
if (latestReleaseConfigurations == null) {
latestReleaseConfigurations = new HashMap<>();
}

if (historyConfigurations == null) {
historyConfigurations = new HashMap<>();
}

Set<String> previousKeys = historyConfigurations.keySet();
Set<String> currentKeys = latestReleaseConfigurations.keySet();

Set<String> commonKeys = Sets.intersection(previousKeys, currentKeys);
Set<String> newKeys = Sets.difference(currentKeys, commonKeys);
Set<String> removedKeys = Sets.difference(previousKeys, commonKeys);

List<ConfigurationChange> changes = Lists.newArrayList();

for (String newKey : newKeys) {
changes.add(new ConfigurationChange(newKey, latestReleaseConfigurations.get(newKey),
ConfigurationChangeType.ADDED));
}

for (String removedKey : removedKeys) {
changes.add(new ConfigurationChange(removedKey, null, ConfigurationChangeType.DELETED));
}

for (String commonKey : commonKeys) {
String previousValue = historyConfigurations.get(commonKey);
String currentValue = latestReleaseConfigurations.get(commonKey);
if (com.google.common.base.Objects.equal(previousValue, currentValue)) {
continue;
}
changes.add(
new ConfigurationChange(commonKey, currentValue, ConfigurationChangeType.MODIFIED));
}

return changes;
}

@Override
public Map<String, Release> findReleasesByReleaseKeys(Set<String> releaseKeys){
return null;
}

/**
* Find active release by id
*/
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@
/**
* @author Jason Song(song_s@ctrip.com)
*/
public interface ConfigService extends ReleaseMessageListener,IncrementalSyncConfigService {
public interface ConfigService extends ReleaseMessageListener, IncrementalSyncConfigService {

/**
* Load config
Original file line number Diff line number Diff line change
@@ -18,7 +18,6 @@

import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder;
import com.ctrip.framework.apollo.biz.config.BizConfig;
import com.ctrip.framework.apollo.core.dto.ConfigurationChange;
import com.google.common.base.Strings;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -38,15 +37,12 @@

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics;

import java.util.Map;
import java.util.Optional;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import javax.annotation.PostConstruct;
@@ -67,13 +63,13 @@ public class ConfigServiceWithCache extends AbstractConfigService {
private static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById";

protected final ReleaseService releaseService;
protected final ReleaseMessageService releaseMessageService;
private final ReleaseMessageService releaseMessageService;
protected final BizConfig bizConfig;
private final MeterRegistry meterRegistry;

protected LoadingCache<String, ConfigCacheEntry> configCache;
private LoadingCache<String, ConfigCacheEntry> configCache;

protected LoadingCache<Long, Optional<Release>> configIdCache;
private LoadingCache<Long, Optional<Release>> configIdCache;

private ConfigCacheEntry nullConfigCacheEntry;

@@ -126,6 +122,7 @@ protected Release findLatestActiveRelease(String appId, String clusterName, Stri

return cacheEntry.getRelease();
}

private void invalidate(String key) {
configCache.invalidate(key);
Tracer.logEvent(TRACER_EVENT_CACHE_INVALIDATE, key);
@@ -232,7 +229,7 @@ public Optional<Release> load(Long key) throws Exception {

}

protected static class ConfigCacheEntry {
private static class ConfigCacheEntry {
private final long notificationId;
private final Release release;

@@ -249,12 +246,4 @@ public Release getRelease() {
return release;
}
}
@Override
public List<ConfigurationChange> calcConfigurationChanges(Map<String, String> latestReleaseConfigurations, Map<String, String> historyConfigurations){
return null;
}
@Override
public Map<String, Release> findReleasesByReleaseKeys(Set<String> releaseKeys){
return null;
}
}
Loading