Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Provide a way to trigger Git mirroring manually #1041

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Provide a way to trigger Git mirroring manually #1041

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public final class MirrorDto {
private final String id;
private final boolean enabled;
private final String projectName;
@Nullable
private final String schedule;
private final String direction;
private final String localRepo;
Expand All @@ -52,7 +53,7 @@ public final class MirrorDto {
public MirrorDto(@JsonProperty("id") String id,
@JsonProperty("enabled") @Nullable Boolean enabled,
@JsonProperty("projectName") String projectName,
@JsonProperty("schedule") String schedule,
@JsonProperty("schedule") @Nullable String schedule,
@JsonProperty("direction") String direction,
@JsonProperty("localRepo") String localRepo,
@JsonProperty("localPath") String localPath,
Expand All @@ -65,7 +66,7 @@ public MirrorDto(@JsonProperty("id") String id,
this.id = requireNonNull(id, "id");
this.enabled = firstNonNull(enabled, true);
this.projectName = requireNonNull(projectName, "projectName");
this.schedule = requireNonNull(schedule, "schedule");
this.schedule = schedule;
this.direction = requireNonNull(direction, "direction");
this.localRepo = requireNonNull(localRepo, "localRepo");
this.localPath = requireNonNull(localPath, "localPath");
Expand All @@ -92,6 +93,7 @@ public String projectName() {
return projectName;
}

@Nullable
@JsonProperty("schedule")
public String schedule() {
return schedule;
Expand Down Expand Up @@ -155,7 +157,7 @@ public boolean equals(Object o) {
return id.equals(mirrorDto.id) &&
enabled == mirrorDto.enabled &&
projectName.equals(mirrorDto.projectName) &&
schedule.equals(mirrorDto.schedule) &&
Objects.equals(schedule, mirrorDto.schedule) &&
direction.equals(mirrorDto.direction) &&
localRepo.equals(mirrorDto.localRepo) &&
localPath.equals(mirrorDto.localPath) &&
Expand Down
65 changes: 40 additions & 25 deletions ...git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.PushResult;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.TagOpt;
import org.eclipse.jgit.transport.URIish;
Expand All @@ -86,8 +87,11 @@
import com.linecorp.centraldogma.server.MirrorException;
import com.linecorp.centraldogma.server.command.Command;
import com.linecorp.centraldogma.server.command.CommandExecutor;
import com.linecorp.centraldogma.server.command.CommitResult;
import com.linecorp.centraldogma.server.credential.Credential;
import com.linecorp.centraldogma.server.mirror.MirrorDirection;
import com.linecorp.centraldogma.server.mirror.MirrorResult;
import com.linecorp.centraldogma.server.mirror.MirrorStatus;
import com.linecorp.centraldogma.server.storage.StorageException;
import com.linecorp.centraldogma.server.storage.repository.Repository;

Expand Down Expand Up @@ -115,7 +119,7 @@ abstract class AbstractGitMirror extends AbstractMirror {
@Nullable
private IgnoreNode ignoreNode;

AbstractGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction,
AbstractGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction,
Credential credential, Repository localRepo, String localPath,
URI remoteRepoUri, String remotePath, String remoteBranch,
@Nullable String gitignore) {
Expand Down Expand Up @@ -169,14 +173,15 @@ GitWithAuth openGit(File workDir,
}
}

void mirrorLocalToRemote(
MirrorResult mirrorLocalToRemote(
GitWithAuth git, int maxNumFiles, long maxNumBytes) throws GitAPIException, IOException {
// TODO(minwoox): Early return if the remote does not have any updates.
final Ref headBranchRef = getHeadBranchRef(git);
final String headBranchRefName = headBranchRef.getName();
final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName);

final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository();
String description;
try (ObjectReader reader = gitRepository.newObjectReader();
TreeWalk treeWalk = new TreeWalk(reader);
RevWalk revWalk = new RevWalk(reader)) {
Expand All @@ -190,9 +195,12 @@ void mirrorLocalToRemote(
final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath);
if (localHead.equals(remoteCurrentRevision)) {
// The remote repository is up-to date.
logger.debug("The remote repository '{}#{}' already at {}. Local repository: '{}'",
remoteRepoUri(), remoteBranch(), localHead, localRepo().name());
return;
description = String.format(
"The remote repository '%s#%s' already at %s. Local repository: '%s/%s'",
remoteRepoUri(), remoteBranch(), localHead,
localRepo().parent().name(), localRepo().name());
logger.debug(description);
return newMirrorResult(MirrorStatus.UP_TO_DATE, description);
}

// Reset to traverse the tree from the first.
Expand All @@ -217,19 +225,27 @@ dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'.
Jackson.writeValueAsPrettyString(mirrorState) + '\n'));
}

final String summary = "Mirror '" + localRepo().name() + "' at " + localHead +
" to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n";
description = summary;
final ObjectId nextCommitId =
commit(gitRepository, dirCache, headCommitId, localHead);
commit(gitRepository, dirCache, headCommitId, summary);
logger.info(summary);
updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId);
}

git.push()
.setRefSpecs(new RefSpec(headBranchRefName))
.setAtomic(true)
.setTimeout(GIT_TIMEOUT_SECS)
.call();
final Iterable<PushResult> pushResults =
git.push()
.setRefSpecs(new RefSpec(headBranchRefName))
.setAtomic(true)
.setTimeout(GIT_TIMEOUT_SECS)
.call();
final PushResult pushResult = pushResults.iterator().next();
// TODO(ikhoon): Append remove ref to description;
return newMirrorResult(MirrorStatus.SUCCESS, description);
}

void mirrorRemoteToLocal(
MirrorResult mirrorRemoteToLocal(
GitWithAuth git, CommandExecutor executor, int maxNumFiles, long maxNumBytes) throws Exception {
final String summary;
final String detail;
Expand All @@ -239,7 +255,13 @@ void mirrorRemoteToLocal(
final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME;
final Revision localRev = localRepo().normalizeNow(Revision.HEAD);
if (!needsFetch(headBranchRef, mirrorStatePath, localRev)) {
return;
final String abbrId = headBranchRef.getObjectId().abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name();
final String message = String.format("Repository '%s/%s' already at %s, %s#%s",
localRepo().parent().name(), localRepo().name(), abbrId,
remoteRepoUri(), remoteBranch());
// The local repository is up-to date.
logger.debug(message);
return newMirrorResult(MirrorStatus.UP_TO_DATE, message);
}

// Update the head commit ID again because there's a chance a commit is pushed between the
Expand Down Expand Up @@ -301,14 +323,12 @@ void mirrorRemoteToLocal(

if (++numFiles > maxNumFiles) {
throwMirrorException(maxNumFiles, "files");
return;
}

final ObjectId objectId = treeWalk.getObjectId(0);
final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY);
if (numBytes > maxNumBytes - contentLength) {
throwMirrorException(maxNumBytes, "bytes");
return;
}
numBytes += contentLength;

Expand Down Expand Up @@ -337,9 +357,11 @@ void mirrorRemoteToLocal(
}
});

executor.execute(Command.push(
final CommitResult commitResult = executor.execute(Command.push(
MIRROR_AUTHOR, localRepo().parent().name(), localRepo().name(),
Revision.HEAD, summary, detail, Markup.PLAINTEXT, changes.values())).join();
final String description = summary + ", Revision: " + commitResult.revision();
return newMirrorResult(MirrorStatus.SUCCESS, description);
}

private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision localRev)
Expand All @@ -355,9 +377,6 @@ private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision l

final ObjectId headCommitId = headBranchRef.getObjectId();
if (headCommitId.name().equals(localSourceRevision)) {
final String abbrId = headCommitId.abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name();
logger.info("Repository '{}' already at {}, {}#{}", localRepo().name(), abbrId,
remoteRepoUri(), remoteBranch());
return false;
}
return true;
Expand Down Expand Up @@ -675,7 +694,7 @@ private static String sanitizeText(String text) {
}

private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache,
ObjectId headCommitId, Revision localHead) throws IOException {
ObjectId headCommitId, String message) throws IOException {
try (ObjectInserter inserter = gitRepository.newObjectInserter()) {
// flush the current index to repository and get the result tree object id.
final ObjectId nextTreeId = dirCache.writeTree(inserter);
Expand All @@ -691,11 +710,7 @@ private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache
commitBuilder.setTreeId(nextTreeId);
commitBuilder.setEncoding(UTF_8);
commitBuilder.setParentId(headCommitId);

final String summary = "Mirror '" + localRepo().name() + "' at " + localHead +
" to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n";
logger.info(summary);
commitBuilder.setMessage(summary);
commitBuilder.setMessage(message);

final ObjectId nextCommitId = inserter.insert(commitBuilder);
inserter.flush();
Expand Down
13 changes: 7 additions & 6 deletions ...-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,14 @@
import com.linecorp.centraldogma.server.internal.credential.AccessTokenCredential;
import com.linecorp.centraldogma.server.internal.credential.PasswordCredential;
import com.linecorp.centraldogma.server.mirror.MirrorDirection;
import com.linecorp.centraldogma.server.mirror.MirrorResult;
import com.linecorp.centraldogma.server.storage.repository.Repository;

final class DefaultGitMirror extends AbstractGitMirror {

private static final Consumer<TransportCommand<?, ?>> NOOP_CONFIGURATOR = command -> {};

DefaultGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction,
DefaultGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction,
Credential credential, Repository localRepo, String localPath,
URI remoteRepoUri, String remotePath, String remoteBranch,
@Nullable String gitignore) {
Expand All @@ -51,9 +52,9 @@ final class DefaultGitMirror extends AbstractGitMirror {
}

@Override
protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception {
protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception {
try (GitWithAuth git = openGit(workDir, transportCommandConfigurator())) {
mirrorLocalToRemote(git, maxNumFiles, maxNumBytes);
return mirrorLocalToRemote(git, maxNumFiles, maxNumBytes);
}
}

Expand All @@ -78,10 +79,10 @@ protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumByt
}

@Override
protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
try (GitWithAuth git = openGit(workDir, transportCommandConfigurator())) {
mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes);
return mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes);
}
}

Expand Down
14 changes: 8 additions & 6 deletions ...rror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import com.linecorp.centraldogma.server.internal.credential.PasswordCredential;
import com.linecorp.centraldogma.server.internal.credential.PublicKeyCredential;
import com.linecorp.centraldogma.server.mirror.MirrorDirection;
import com.linecorp.centraldogma.server.mirror.MirrorResult;
import com.linecorp.centraldogma.server.storage.repository.Repository;

final class SshGitMirror extends AbstractGitMirror {
Expand All @@ -79,7 +80,7 @@ final class SshGitMirror extends AbstractGitMirror {
// We might create multiple BouncyCastleRandom later and poll them, if necessary.
private static final BouncyCastleRandom bounceCastleRandom = new BouncyCastleRandom();

SshGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction,
SshGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction,
Credential credential, Repository localRepo, String localPath,
URI remoteRepoUri, String remotePath, String remoteBranch,
@Nullable String gitignore) {
Expand All @@ -89,28 +90,29 @@ final class SshGitMirror extends AbstractGitMirror {
}

@Override
protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception {
protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes)
throws Exception {
final URIish remoteUri = remoteUri();
try (SshClient sshClient = createSshClient();
ClientSession session = createSession(sshClient, remoteUri)) {
final DefaultGitSshdSessionFactory sessionFactory =
new DefaultGitSshdSessionFactory(sshClient, session);
try (GitWithAuth git = openGit(workDir, remoteUri, sessionFactory::configureCommand)) {
mirrorLocalToRemote(git, maxNumFiles, maxNumBytes);
return mirrorLocalToRemote(git, maxNumFiles, maxNumBytes);
}
}
}

@Override
protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
final URIish remoteUri = remoteUri();
try (SshClient sshClient = createSshClient();
ClientSession session = createSession(sshClient, remoteUri)) {
final DefaultGitSshdSessionFactory sessionFactory =
new DefaultGitSshdSessionFactory(sshClient, session);
try (GitWithAuth git = openGit(workDir, remoteUri, sessionFactory::configureCommand)) {
mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes);
return mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes);
}
}
}
Expand Down
11 changes: 8 additions & 3 deletions ...st/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import com.linecorp.centraldogma.server.credential.Credential;
import com.linecorp.centraldogma.server.mirror.Mirror;
import com.linecorp.centraldogma.server.mirror.MirrorDirection;
import com.linecorp.centraldogma.server.mirror.MirrorResult;
import com.linecorp.centraldogma.server.mirror.MirrorStatus;
import com.linecorp.centraldogma.server.storage.project.Project;
import com.linecorp.centraldogma.server.storage.project.ProjectManager;
import com.linecorp.centraldogma.server.storage.repository.MetaRepository;
Expand Down Expand Up @@ -74,14 +76,17 @@ void mirroringTaskShouldNeverBeRejected() {
Credential.FALLBACK, r, "/",
URI.create("unused://uri"), "/", "", null) {
@Override
protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) {}
protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) {
return newMirrorResult(MirrorStatus.UP_TO_DATE, null);
}

@Override
protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor,
int maxNumFiles, long maxNumBytes) throws Exception {
// Sleep longer than mirroring interval so that the workers fall behind.
taskCounter.incrementAndGet();
Thread.sleep(2000);
return newMirrorResult(MirrorStatus.SUCCESS, null);
}
};

Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,7 +813,8 @@ private void configureHttpApi(ServerBuilder sb,
.annotatedService(new RepositoryServiceV1(executor, mds));

if (GIT_MIRROR_ENABLED) {
apiV1ServiceBuilder.annotatedService(new MirroringServiceV1(projectApiManager, executor))
apiV1ServiceBuilder.annotatedService(
new MirroringServiceV1(projectApiManager, executor, cfg.dataDir()))
.annotatedService(new CredentialServiceV1(projectApiManager, executor));
}

Expand Down
Loading
Loading