diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 96c841118c..d16a7b9660 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -130,6 +130,28 @@ jobs: class: org.apache.streampark.e2e.cases.TokenManagementTest - name: UploadManagementTest class: org.apache.streampark.e2e.cases.UploadManagementTest + - name: ProjectsManagementTest + class: org.apache.streampark.e2e.cases.ProjectsManagementTest + - name: VariableManagementTest + class: org.apache.streampark.e2e.cases.VariableManagementTest + - name: Flink118OnRemoteClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink118OnRemoteClusterDeployTest + - name: Flink117OnRemoteClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink117OnRemoteClusterDeployTest + - name: Flink116OnRemoteClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink116OnRemoteClusterDeployTest + - name: Flink116OnYarnClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink116OnYarnClusterDeployTest + - name: Flink117OnYarnClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink117OnYarnClusterDeployTest + - name: Flink118OnYarnClusterDeployTest + class: org.apache.streampark.e2e.cases.Flink118OnYarnClusterDeployTest + - name: FlinkSQL116OnYarnTest + class: org.apache.streampark.e2e.cases.FlinkSQL116OnYarnTest + - name: FlinkSQL117OnYarnTest + class: org.apache.streampark.e2e.cases.FlinkSQL117OnYarnTest + - name: FlinkSQL118OnYarnTest + class: org.apache.streampark.e2e.cases.FlinkSQL118OnYarnTest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java index fed5aea87d..49d8a6ba12 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java @@ -17,21 +17,19 @@ package org.apache.streampark.console.base.util; -import org.apache.streampark.common.util.FileUtils; -import org.apache.streampark.common.util.SystemPropertyUtils; -import org.apache.streampark.console.core.entity.Project; - -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; +import lombok.Getter; +import lombok.Setter; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.LsRemoteCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.InvalidRemoteException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.transport.JschConfigSessionFactory; @@ -40,102 +38,219 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.util.FS; +import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -/** used to build project and project build task */ +/** */ public class GitUtils { private GitUtils() { } - public static Git clone(Project project) throws GitAPIException { - CloneCommand cloneCommand = Git.cloneRepository().setURI(project.getUrl()).setDirectory(project.getAppSource()); - - if (StringUtils.isNotBlank(project.getBranches())) { - cloneCommand.setBranch(Constants.R_HEADS + project.getBranches()); - cloneCommand.setBranchesToClone( - Collections.singletonList(Constants.R_HEADS + project.getBranches())); + public static Git clone(GitCloneRequest request) throws GitAPIException { + try { + CloneCommand cloneCommand = + Git.cloneRepository().setURI(request.getUrl()).setDirectory(request.getStoreDir()); + setCredentials(cloneCommand, request); + if (StringUtils.isNotBlank(request.getBranch())) { + cloneCommand.setBranch(Constants.R_HEADS + request.getBranch()); + cloneCommand.setBranchesToClone( + Collections.singletonList(Constants.R_HEADS + request.getBranch())); + } + Git git = cloneCommand.call(); + if (StringUtils.isNotBlank(request.getBranch())) { + git.checkout().setName(request.getBranch()).call(); + } else if (StringUtils.isNotBlank(request.getTag())) { + git.checkout().setName(request.getTag()).call(); + } else { + throw new IllegalArgumentException("git clone failed, No tag or branch specified"); + } + return git; + } catch (Exception e) { + if (e instanceof InvalidRemoteException && request.getConnType() == GitConnType.HTTP) { + String url = httpUrlToSSH(request.getUrl()); + request.setUrl(url); + return clone(request); + } + throw e; } - setCredentials(cloneCommand, project); - return cloneCommand.call(); } - public static List getBranchList(Project project) throws GitAPIException { - LsRemoteCommand command = Git.lsRemoteRepository().setRemote(project.getUrl()).setHeads(true); - setCredentials(command, project); - Collection refList = command.call(); - List branchList = new ArrayList<>(4); - if (CollectionUtils.isEmpty(refList)) { + public static List getBranches(GitGetRequest request) throws GitAPIException { + try { + LsRemoteCommand command = Git.lsRemoteRepository().setRemote(request.getUrl()).setHeads(true); + setCredentials(command, request); + Collection refList = command.call(); + List branchList = new ArrayList<>(4); + for (Ref ref : refList) { + String refName = ref.getName(); + if (refName.startsWith(Constants.R_HEADS)) { + String branchName = refName.replace(Constants.R_HEADS, ""); + branchList.add(branchName); + } + } return branchList; + } catch (Exception e) { + if (e instanceof InvalidRemoteException && request.getConnType() == GitConnType.HTTP) { + String url = httpUrlToSSH(request.getUrl()); + request.setUrl(url); + return getBranches(request); + } + throw e; } - for (Ref ref : refList) { - String refName = ref.getName(); - if (refName.startsWith(Constants.R_HEADS)) { - String branchName = refName.replace(Constants.R_HEADS, ""); - branchList.add(branchName); + } + + public static List getTags(GitGetRequest request) throws GitAPIException { + try { + LsRemoteCommand command = Git.lsRemoteRepository().setRemote(request.getUrl()).setTags(true); + setCredentials(command, request); + Collection refList = command.call(); + List tagList = new ArrayList<>(4); + for (Ref ref : refList) { + String refName = ref.getName(); + if (refName.startsWith(Constants.R_TAGS)) { + String branchName = refName.replace(Constants.R_TAGS, ""); + tagList.add(branchName); + } + } + return tagList; + } catch (Exception e) { + if (e instanceof InvalidRemoteException && request.getConnType() == GitConnType.HTTP) { + String url = httpUrlToSSH(request.getUrl()); + request.setUrl(url); + return getTags(request); } + throw e; } - return branchList; } - private static void setCredentials(TransportCommand transportCommand, Project project) { - if (project.isHttpRepositoryUrl()) { - if (!StringUtils.isAllEmpty(project.getUserName(), project.getPassword())) { - try { - String decrypt = StringUtils.isNotBlank(project.getSalt()) - ? EncryptUtils.decrypt(project.getPassword(), project.getSalt()) - : project.getPassword(); - UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider( - project.getUserName(), decrypt); + public static String httpUrlToSSH(String url) { + return url.replaceAll("(https://|http://)(.*?)/(.*?)/(.*?)(\\.git|)\\s*$", "git@$2:$3/$4.git"); + } + + public static boolean isSshRepositoryUrl(String url) { + return url.trim().startsWith("git@"); + } + + public static boolean isHttpRepositoryUrl(String url) { + return !isSshRepositoryUrl(url); + } + + private static void setCredentials( + TransportCommand transportCommand, GitAuthRequest request) { + switch (request.connType) { + case HTTP: + if (!StringUtils.isAllEmpty(request.getUsername(), request.getPassword())) { + UsernamePasswordCredentialsProvider credentialsProvider = + new UsernamePasswordCredentialsProvider(request.getUsername(), request.getPassword()); transportCommand.setCredentialsProvider(credentialsProvider); - } catch (Exception e) { - throw new IllegalStateException( - "[StreamPark] git setCredentials: project password decrypt failed", e); } - } - } else if (project.isSshRepositoryUrl()) { - transportCommand.setTransportConfigCallback( - transport -> { - SshTransport sshTransport = (SshTransport) transport; - sshTransport.setSshSessionFactory( - new JschConfigSessionFactory() { - - @Override - protected void configure(OpenSshConfig.Host hc, Session session) { - session.setConfig("StrictHostKeyChecking", "no"); - } - - @Override - protected JSch createDefaultJSch(FS fs) throws JSchException { - JSch jSch = super.createDefaultJSch(fs); - String prvkeyPath = project.getPrvkeyPath(); - if (StringUtils.isBlank(prvkeyPath)) { - String userHome = SystemPropertyUtils.getUserHome(); - if (userHome != null) { - String rsaPath = userHome.concat("/.ssh/id_rsa"); - if (FileUtils.exists(rsaPath)) { - prvkeyPath = rsaPath; + break; + case SSH: + transportCommand.setTransportConfigCallback( + transport -> { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory( + new JschConfigSessionFactory() { + + @Override + protected void configure(OpenSshConfig.Host hc, Session session) { + session.setConfig("StrictHostKeyChecking", "no"); + } + + @Override + protected JSch createDefaultJSch(FS fs) throws JSchException { + JSch jSch = super.createDefaultJSch(fs); + String prvkeyPath = request.getPrivateKey(); + if (StringUtils.isBlank(prvkeyPath)) { + String userHome = System.getProperty("user.home"); + if (userHome != null) { + String rsaPath = userHome.concat("/.ssh/id_rsa"); + File resFile = new File(rsaPath); + if (resFile.exists()) { + prvkeyPath = rsaPath; + } } } - } - if (prvkeyPath == null) { + if (prvkeyPath == null) { + return jSch; + } + if (StringUtils.isEmpty(request.getPassword())) { + jSch.addIdentity(prvkeyPath); + } else { + jSch.addIdentity(prvkeyPath, request.getPassword()); + } return jSch; } - if (StringUtils.isBlank(project.getPassword())) { - jSch.addIdentity(prvkeyPath); - } else { - jSch.addIdentity(prvkeyPath, project.getPassword()); - } - return jSch; - } - }); - }); - } else { - throw new IllegalStateException( - "[StreamPark] repository URL is invalid, must be ssh or http(s)"); + }); + }); + break; + default: + throw new IllegalStateException( + "[StreamPark] repository URL is invalid, must be ssh or http(s)"); + } + } + + @Getter + public enum GitConnType { + HTTP, + SSH + } + + @Getter + @Setter + public static class GitAuthRequest { + + private GitConnType connType; + private String username; + private String password; + private String privateKey; + } + + @Getter + @Setter + public static class GitGetRequest extends GitAuthRequest { + + private String url; + + public void setUrl(String url) { + if (StringUtils.isBlank(url)) { + throw new IllegalArgumentException("git url cannot be empty"); + } + this.url = url; + if (GitUtils.isSshRepositoryUrl(url)) { + setConnType(GitConnType.SSH); + } else { + setConnType(GitConnType.HTTP); + } + } + } + + @Getter + @Setter + public static class GitCloneRequest extends GitGetRequest { + + private File storeDir; + private String branch; + private String tag; + + public void setRefs(String refs) { + if (StringUtils.isNotBlank(refs)) { + if (!refs.startsWith(Constants.R_REFS)) { + this.branch = refs; + return; + } + if (refs.startsWith(Constants.R_HEADS)) { + this.branch = refs.replace(Constants.R_HEADS, ""); + return; + } + if (refs.startsWith(Constants.R_TAGS)) { + this.tag = refs.replace(Constants.R_TAGS, ""); + } + } } } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/ObjectUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/ObjectUtils.java index 0273573aad..2dbeb3d011 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/ObjectUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/ObjectUtils.java @@ -17,6 +17,8 @@ package org.apache.streampark.console.base.util; +import java.lang.reflect.Array; +import java.util.Arrays; import java.util.Objects; public final class ObjectUtils { @@ -24,6 +26,896 @@ public final class ObjectUtils { private ObjectUtils() { } + private static final int INITIAL_HASH = 7; + private static final int MULTIPLIER = 31; + + private static final String EMPTY_STRING = ""; + private static final String NULL_STRING = "null"; + private static final String ARRAY_START = "{"; + private static final String ARRAY_END = "}"; + private static final String EMPTY_ARRAY = ARRAY_START + ARRAY_END; + private static final String ARRAY_ELEMENT_SEPARATOR = ", "; + + /** + * Return whether the given throwable is a checked exception: that is, neither a RuntimeException + * nor an Error. + * + * @param ex the throwable to check + * @return whether the throwable is a checked exception + * @see Exception + * @see RuntimeException + * @see Error + */ + public static boolean isCheckedException(Throwable ex) { + return !(ex instanceof RuntimeException || ex instanceof Error); + } + + /** + * Check whether the given exception is compatible with the exceptions declared in a throws + * clause. + * + * @param ex the exception to checked + * @param declaredExceptions the exceptions declared in the throws clause + * @return whether the given exception is compatible + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static boolean isCompatibleWithThrowsClause(Throwable ex, Class[] declaredExceptions) { + if (!isCheckedException(ex)) { + return true; + } + if (declaredExceptions != null) { + int i = 0; + while (i < declaredExceptions.length) { + if (declaredExceptions[i].isAssignableFrom(ex.getClass())) { + return true; + } + i++; + } + } + return false; + } + + /** + * Determine whether the given object is an array: either an Object array or a primitive array. + * + * @param obj the object to check + */ + public static boolean isArray(Object obj) { + return (obj != null && obj.getClass().isArray()); + } + + /** + * Determine whether the given array is empty: i.e. null or of zero length. + * + * @param array the array to check + */ + public static boolean isEmpty(Object[] array) { + return (array == null || array.length == 0); + } + + /** + * Check whether the given array contains the given element. + * + * @param array the array to check (may be null, in which case the return value will + * always be false) + * @param element the element to check for + * @return whether the element has been found in the given array + */ + public static boolean containsElement(Object[] array, Object element) { + if (array == null) { + return false; + } + for (Object arrayEle : array) { + if (safeEquals(arrayEle, element)) { + return true; + } + } + return false; + } + + /** + * Check whether the given array of enum constants contains a constant with the given name, + * ignoring case when determining a match. + * + * @param enumValues the enum values to check, typically the product of a call to MyEnum.values() + * @param constant the constant name to find (must not be null or empty string) + * @return whether the constant has been found in the given array + */ + public static boolean containsConstant(Enum[] enumValues, String constant) { + return containsConstant(enumValues, constant, false); + } + + /** + * Check whether the given array of enum constants contains a constant with the given name. + * + * @param enumValues the enum values to check, typically the product of a call to MyEnum.values() + * @param constant the constant name to find (must not be null or empty string) + * @param caseSensitive whether case is significant in determining a match + * @return whether the constant has been found in the given array + */ + public static boolean containsConstant( + Enum[] enumValues, String constant, boolean caseSensitive) { + for (Enum candidate : enumValues) { + if (caseSensitive + ? candidate.toString().equals(constant) + : candidate.toString().equalsIgnoreCase(constant)) { + return true; + } + } + return false; + } + + /** + * Case insensitive alternative to {@link Enum#valueOf(Class, String)}. + * + * @param the concrete Enum type + * @param enumValues the array of all Enum constants in question, usually per Enum.values() + * @param constant the constant to get the enum value of + * @throws IllegalArgumentException if the given constant is not found in the given array of enum + * values. Use {@link #containsConstant(Enum[], String)} as a guard to avoid this exception. + */ + public static > E caseInsensitiveValueOf(E[] enumValues, String constant) { + for (E candidate : enumValues) { + if (candidate.toString().equalsIgnoreCase(constant)) { + return candidate; + } + } + throw new IllegalArgumentException( + String.format( + "constant [%s] does not exist in enum type %s", + constant, enumValues.getClass().getComponentType().getName())); + } + + /** + * Append the given object to the given array, returning a new array consisting of the input array + * contents plus the given object. + * + * @param array the array to append to (can be null) + * @param obj the object to append + * @return the new array (of the same component type; never null) + */ + public static A[] addObjectToArray(A[] array, O obj) { + Class compType = Object.class; + if (array != null) { + compType = array.getClass().getComponentType(); + } else if (obj != null) { + compType = obj.getClass(); + } + int newArrLength = (array != null ? array.length + 1 : 1); + @SuppressWarnings("unchecked") + A[] newArr = (A[]) Array.newInstance(compType, newArrLength); + if (array != null) { + System.arraycopy(array, 0, newArr, 0, array.length); + } + newArr[newArr.length - 1] = obj; + return newArr; + } + + /** + * Convert the given array (which may be a primitive array) to an object array (if necessary of + * primitive wrapper objects). + * + *

A null source value will be converted to an empty Object array. + * + * @param source the (potentially primitive) array + * @return the corresponding object array (never null) + * @throws IllegalArgumentException if the parameter is not an array + */ + public static Object[] toObjectArray(Object source) { + if (source instanceof Object[]) { + return (Object[]) source; + } + if (source == null) { + return new Object[0]; + } + if (!source.getClass().isArray()) { + throw new IllegalArgumentException("Source is not an array: " + source); + } + int length = Array.getLength(source); + if (length == 0) { + return new Object[0]; + } + @SuppressWarnings("rawtypes") + Class wrapperType = Array.get(source, 0).getClass(); + Object[] newArray = (Object[]) Array.newInstance(wrapperType, length); + for (int i = 0; i < length; i++) { + newArray[i] = Array.get(source, i); + } + return newArray; + } + + // --------------------------------------------------------------------- + // Convenience methods for content-based equality/hash-code handling + // --------------------------------------------------------------------- + + /** + * Determine if the given objects are equal, returning true if both are null + * or false if only one is null. + * + *

Compares arrays with Arrays.equals, performing an equality check based on the + * array elements rather than the array reference. + * + * @param o1 first Object to compare + * @param o2 second Object to compare + * @return whether the given objects are equal + * @see Arrays#equals + */ + public static boolean safeEquals(Object o1, Object o2) { + if (o1 == null || o2 == null) { + return false; + } + + if (o1 == o2) { + return true; + } + + if (o1.equals(o2)) { + return true; + } + if (o1.getClass().isArray() && o2.getClass().isArray()) { + if (o1 instanceof Object[] && o2 instanceof Object[]) { + return Arrays.equals((Object[]) o1, (Object[]) o2); + } + if (o1 instanceof boolean[] && o2 instanceof boolean[]) { + return Arrays.equals((boolean[]) o1, (boolean[]) o2); + } + if (o1 instanceof byte[] && o2 instanceof byte[]) { + return Arrays.equals((byte[]) o1, (byte[]) o2); + } + if (o1 instanceof char[] && o2 instanceof char[]) { + return Arrays.equals((char[]) o1, (char[]) o2); + } + if (o1 instanceof double[] && o2 instanceof double[]) { + return Arrays.equals((double[]) o1, (double[]) o2); + } + if (o1 instanceof float[] && o2 instanceof float[]) { + return Arrays.equals((float[]) o1, (float[]) o2); + } + if (o1 instanceof int[] && o2 instanceof int[]) { + return Arrays.equals((int[]) o1, (int[]) o2); + } + if (o1 instanceof long[] && o2 instanceof long[]) { + return Arrays.equals((long[]) o1, (long[]) o2); + } + if (o1 instanceof short[] && o2 instanceof short[]) { + return Arrays.equals((short[]) o1, (short[]) o2); + } + } + return false; + } + + public static boolean safeTrimEquals(Object o1, Object o2) { + boolean equals = safeEquals(o1, o2); + if (!equals) { + if (o1 != null && o2 != null) { + if (o1 instanceof String && o2 instanceof String) { + return o1.toString().trim().equals(o2.toString().trim()); + } + } + } + return equals; + } + + /** + * Return as hash code for the given object; typically the value of + * {@link Object#hashCode()}. If the object is an array, this method will delegate to any + * of the safeHashCode methods for arrays in this class. If the object is null + * , this method returns 0. + * + * @see #safeHashCode(Object[]) + * @see #safeHashCode(boolean[]) + * @see #safeHashCode(byte[]) + * @see #safeHashCode(char[]) + * @see #safeHashCode(double[]) + * @see #safeHashCode(float[]) + * @see #safeHashCode(int[]) + * @see #safeHashCode(long[]) + * @see #safeHashCode(short[]) + */ + public static int safeHashCode(Object obj) { + if (obj == null) { + return 0; + } + if (obj.getClass().isArray()) { + if (obj instanceof Object[]) { + return safeHashCode((Object[]) obj); + } + if (obj instanceof boolean[]) { + return safeHashCode((boolean[]) obj); + } + if (obj instanceof byte[]) { + return safeHashCode((byte[]) obj); + } + if (obj instanceof char[]) { + return safeHashCode((char[]) obj); + } + if (obj instanceof double[]) { + return safeHashCode((double[]) obj); + } + if (obj instanceof float[]) { + return safeHashCode((float[]) obj); + } + if (obj instanceof int[]) { + return safeHashCode((int[]) obj); + } + if (obj instanceof long[]) { + return safeHashCode((long[]) obj); + } + if (obj instanceof short[]) { + return safeHashCode((short[]) obj); + } + } + return obj.hashCode(); + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(Object[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (Object anArray : array) { + hash = MULTIPLIER * hash + safeHashCode(anArray); + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(boolean[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (boolean anArray : array) { + hash = MULTIPLIER * hash + hashCode(anArray); + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(byte[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (byte anArray : array) { + hash = MULTIPLIER * hash + anArray; + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(char[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (char anArray : array) { + hash = MULTIPLIER * hash + anArray; + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(double[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (double anArray : array) { + hash = MULTIPLIER * hash + hashCode(anArray); + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(float[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (float anArray : array) { + hash = MULTIPLIER * hash + hashCode(anArray); + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(int[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (int anArray : array) { + hash = MULTIPLIER * hash + anArray; + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(long[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (long anArray : array) { + hash = MULTIPLIER * hash + hashCode(anArray); + } + return hash; + } + + /** + * Return a hash code based on the contents of the specified array. If array is + * null, this method returns 0. + */ + public static int safeHashCode(short[] array) { + if (array == null) { + return 0; + } + int hash = INITIAL_HASH; + for (short anArray : array) { + hash = MULTIPLIER * hash + anArray; + } + return hash; + } + + /** + * Return the same value as {@link Boolean#hashCode()}. + * + * @see Boolean#hashCode() + */ + public static int hashCode(boolean bool) { + return bool ? 1231 : 1237; + } + + /** + * Return the same value as {@link Double#hashCode()}. + * + * @see Double#hashCode() + */ + public static int hashCode(double dbl) { + long bits = Double.doubleToLongBits(dbl); + return hashCode(bits); + } + + /** + * Return the same value as {@link Float#hashCode()}. + * + * @see Float#hashCode() + */ + public static int hashCode(float flt) { + return Float.floatToIntBits(flt); + } + + /** + * Return the same value as {@link Long#hashCode()}. + * + * @see Long#hashCode() + */ + public static int hashCode(long lng) { + return (int) (lng ^ (lng >>> 32)); + } + + // --------------------------------------------------------------------- + // Convenience methods for toString output + // --------------------------------------------------------------------- + + /** + * Return a String representation of an object's overall identity. + * + * @param obj the object (may be null) + * @return the object's identity as String representation, or an empty String if the object was + * null + */ + public static String identityToString(Object obj) { + if (obj == null) { + return EMPTY_STRING; + } + return obj.getClass().getName() + "@" + getIdentityHexString(obj); + } + + /** + * Return a hex String form of an object's identity hash code. + * + * @param obj the object + * @return the object's identity code in hex notation + */ + public static String getIdentityHexString(Object obj) { + return Integer.toHexString(System.identityHashCode(obj)); + } + + /** + * Return a content-based String representation if obj is not null; + * otherwise returns an empty String. + * + *

Differs from {@link #safeToString(Object)} in that it returns an empty String rather than + * "null" for a null value. + * + * @param obj the object to build a display String for + * @return a display String representation of obj + * @see #safeToString(Object) + */ + public static String getDisplayString(Object obj) { + if (obj == null) { + return EMPTY_STRING; + } + return safeToString(obj); + } + + /** + * Determine the class name for the given object. + * + *

Returns "null" if obj is null. + * + * @param obj the object to introspect (may be null) + * @return the corresponding class name + */ + public static String safeClassName(Object obj) { + return (obj != null ? obj.getClass().getName() : NULL_STRING); + } + + /** + * Return a String representation of the specified Object. + * + *

Builds a String representation of the contents in case of an array. Returns "null" + * if obj is null. + * + * @param obj the object to build a String representation for + * @return a String representation of obj + */ + public static String safeToString(Object obj) { + if (obj == null) { + return NULL_STRING; + } + if (obj instanceof String) { + return (String) obj; + } + if (obj instanceof Object[]) { + return safeToString((Object[]) obj); + } + if (obj instanceof boolean[]) { + return safeToString((boolean[]) obj); + } + if (obj instanceof byte[]) { + return safeToString((byte[]) obj); + } + if (obj instanceof char[]) { + return safeToString((char[]) obj); + } + if (obj instanceof double[]) { + return safeToString((double[]) obj); + } + if (obj instanceof float[]) { + return safeToString((float[]) obj); + } + if (obj instanceof int[]) { + return safeToString((int[]) obj); + } + if (obj instanceof long[]) { + return safeToString((long[]) obj); + } + if (obj instanceof short[]) { + return safeToString((short[]) obj); + } + String str = obj.toString(); + return (str != null ? str : EMPTY_STRING); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(Object[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(boolean[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(byte[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(char[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append("'").append(array[i]).append("'"); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(double[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(float[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(int[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(long[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + + /** + * Return a String representation of the contents of the specified array. + * + *

The String representation consists of a list of the array's elements, enclosed in curly + * braces ("{}"). Adjacent elements are separated by the characters ", " + * (a comma followed by a space). Returns "null" if array is null + * . + * + * @param array the array to build a String representation for + * @return a String representation of array + */ + public static String safeToString(short[] array) { + if (array == null) { + return NULL_STRING; + } + int length = array.length; + if (length == 0) { + return EMPTY_ARRAY; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i == 0) { + sb.append(ARRAY_START); + } else { + sb.append(ARRAY_ELEMENT_SEPARATOR); + } + sb.append(array[i]); + } + sb.append(ARRAY_END); + return sb.toString(); + } + public static boolean trimEquals(Object o1, Object o2) { boolean equals = Objects.deepEquals(o1, o2); if (!equals && o1 instanceof String && o2 instanceof String) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java index 50b33a7a16..1f09817580 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java @@ -38,6 +38,7 @@ import org.springframework.web.bind.annotation.RestController; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -105,8 +106,12 @@ public RestResponse list(Project project, RestRequest restRequest) { @PostMapping("branches") @Permission(team = "#project.teamId") public RestResponse branches(Project project) { - List branches = project.getAllBranches(); - return RestResponse.success().data(branches); + List branches = projectService.getAllBranches(project); + List tags = projectService.getAllTags(project); + Map> refs = new HashMap<>(); + refs.put("tags", tags); + refs.put("branches", branches); + return RestResponse.success().data(refs); } @PostMapping("delete") @@ -120,7 +125,7 @@ public RestResponse delete(Project project) { @PostMapping("git_check") @Permission(team = "#project.teamId") public RestResponse gitCheck(Project project) { - GitAuthorizedErrorEnum error = project.gitCheck(); + GitAuthorizedErrorEnum error = projectService.gitCheck(project); return RestResponse.success().data(error.getType()); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java index b6d276052e..6858a80c82 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java @@ -36,7 +36,6 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.core.toolkit.support.SFunction; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -123,10 +122,12 @@ public FlinkK8sRestExposedType getK8sRestExposedTypeEnum() { return FlinkK8sRestExposedType.of(this.k8sRestExposedType); } + @JsonIgnore public FlinkExecutionMode getFlinkExecutionModeEnum() { return FlinkExecutionMode.of(this.executionMode); } + @JsonIgnore public ClusterState getClusterStateEnum() { return ClusterState.of(this.clusterState); } @@ -188,13 +189,4 @@ public Map getProperties() { return propertyMap; } - public static class SFunc { - - public static final SFunction ID = FlinkCluster::getId; - public static final SFunction ADDRESS = FlinkCluster::getAddress; - public static final SFunction JOB_MANAGER_URL = FlinkCluster::getJobManagerUrl; - public static final SFunction CLUSTER_STATE = FlinkCluster::getClusterState; - public static final SFunction EXECUTION_MODE = FlinkCluster::getExecutionMode; - public static final SFunction EXCEPTION = FlinkCluster::getException; - } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java index 8d354de6af..79a23630eb 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java @@ -22,11 +22,7 @@ import org.apache.streampark.common.conf.Workspace; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.console.base.exception.ApiDetailException; -import org.apache.streampark.console.base.mybatis.entity.BaseEntity; -import org.apache.streampark.console.base.util.GitUtils; import org.apache.streampark.console.base.util.WebUtils; -import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -37,25 +33,25 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Data; -import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.eclipse.jgit.lib.Constants; import java.io.File; import java.io.IOException; +import java.io.Serializable; import java.util.Arrays; import java.util.Date; import java.util.Iterator; -import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @Slf4j -@Data -@EqualsAndHashCode(callSuper = true) +@Getter +@Setter @TableName("t_flink_project") -public class Project extends BaseEntity { +public class Project implements Serializable { @TableId(type = IdType.AUTO) private Long id; @@ -69,9 +65,6 @@ public class Project extends BaseEntity { /** git branch or tag */ private String refs; - /** git branch */ - private String branches; - private Date lastBuild; @TableField(updateStrategy = FieldStrategy.IGNORED) @@ -83,10 +76,6 @@ public class Project extends BaseEntity { @TableField(updateStrategy = FieldStrategy.IGNORED) private String prvkeyPath; - /** No salt value is returned */ - @JsonIgnore - private String salt; - /** 1:git 2:svn */ private Integer repository; @@ -104,6 +93,10 @@ public class Project extends BaseEntity { /** 1) flink 2) spark */ private Integer type; + private Date createTime; + + private Date modifyTime; + private transient String module; private transient String dateFrom; @@ -124,7 +117,8 @@ public File getAppSource() { } String sourceDir = getSourceDirName(); - File srcFile = new File(String.format("%s/%s/%s", sourcePath.getAbsolutePath(), name, sourceDir)); + File srcFile = + new File(String.format("%s/%s/%s", sourcePath.getAbsolutePath(), name, sourceDir)); String newPath = String.format("%s/%s", sourcePath.getAbsolutePath(), id); if (srcFile.exists()) { File newFile = new File(newPath); @@ -138,7 +132,16 @@ public File getAppSource() { } private String getSourceDirName() { - String branches = this.getBranches() == null ? "main" : this.getBranches(); + String branches = "main"; + if (StringUtils.isNotBlank(this.refs)) { + if (this.refs.startsWith(Constants.R_HEADS)) { + branches = this.refs.replace(Constants.R_HEADS, ""); + } else if (this.refs.startsWith(Constants.R_TAGS)) { + branches = this.refs.replace(Constants.R_TAGS, ""); + } else { + branches = this.refs; + } + } String rootName = url.replaceAll(".*/|\\.git|\\.svn", ""); return rootName.concat("-").concat(branches); } @@ -159,30 +162,6 @@ public void delete() throws IOException { FileUtils.deleteDirectory(getDistHome()); } - @JsonIgnore - public List getAllBranches() { - try { - return GitUtils.getBranchList(this); - } catch (Exception e) { - throw new ApiDetailException(e); - } - } - - public GitAuthorizedErrorEnum gitCheck() { - try { - GitUtils.getBranchList(this); - return GitAuthorizedErrorEnum.SUCCESS; - } catch (Exception e) { - String err = e.getMessage(); - if (err.contains("not authorized")) { - return GitAuthorizedErrorEnum.ERROR; - } else if (err.contains("Authentication is required")) { - return GitAuthorizedErrorEnum.REQUIRED; - } - return GitAuthorizedErrorEnum.UNKNOW; - } - } - @JsonIgnore public boolean isCloned() { File repository = getGitRepository(); @@ -202,39 +181,47 @@ public void cleanCloned() throws IOException { @JsonIgnore public String getMavenArgs() { - // 1) check build args - String buildArg = getMvnBuildArgs(); - StringBuilder argBuilder = new StringBuilder(); - if (StringUtils.isNotBlank(buildArg)) { - argBuilder.append(buildArg); + StringBuilder mvnArgBuffer = new StringBuilder(" clean package -DskipTests "); + if (StringUtils.isNotBlank(this.buildArgs)) { + mvnArgBuffer.append(this.buildArgs.trim()); } - // 2) mvn setting file - String mvnSetting = getMvnSetting(); - if (StringUtils.isNotBlank(mvnSetting)) { - argBuilder.append(" --settings ").append(mvnSetting); + // --settings + String setting = InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH()); + if (StringUtils.isNotBlank(setting)) { + File file = new File(setting); + if (file.exists() && file.isFile()) { + mvnArgBuffer.append(" --settings ").append(setting.trim()); + } else { + throw new IllegalArgumentException( + String.format( + "Invalid maven-setting file path \"%s\", the path not exist or is not file", + setting)); + } } - // 3) check args - String cmd = argBuilder.toString(); - String illegalArg = getIllegalArgs(cmd); - if (illegalArg != null) { + // check maven args + String mvnArgs = mvnArgBuffer.toString(); + if (mvnArgs.contains("\n")) { throw new IllegalArgumentException( String.format( - "Invalid maven argument, illegal args: %s, in your maven args: %s", - illegalArg, cmd)); + "Illegal argument: newline character in maven build parameters: \"%s\"", mvnArgs)); } - String mvn = getMvn(); - return mvn.concat(" clean package -DskipTests ").concat(cmd); - } + String args = getIllegalArgs(mvnArgs); + if (args != null) { + throw new IllegalArgumentException( + String.format("Illegal argument: \"%s\" in maven build parameters: %s", args, mvnArgs)); + } - private String getMvn() { + // find mvn boolean windows = Utils.isWindows(); String mvn = windows ? "mvn.cmd" : "mvn"; String mavenHome = System.getenv("M2_HOME"); - mavenHome = mavenHome == null ? System.getenv("MAVEN_HOME") : mavenHome; + if (mavenHome == null) { + mavenHome = System.getenv("MAVEN_HOME"); + } boolean useWrapper = true; if (mavenHome != null) { @@ -250,36 +237,13 @@ private String getMvn() { } if (useWrapper) { - mvn = WebUtils.getAppHome().concat(windows ? "/bin/mvnw.cmd" : "/bin/mvnw"); - } - return mvn; - } - - private String getMvnBuildArgs() { - if (StringUtils.isNotBlank(this.buildArgs)) { - String args = getIllegalArgs(this.buildArgs); - AssertUtils.required( - args == null, - String.format( - "Illegal argument: \"%s\" in maven build parameters: %s", args, - this.buildArgs)); - return this.buildArgs.trim(); - } - return null; - } - - private String getMvnSetting() { - String setting = InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH()); - if (StringUtils.isBlank(setting)) { - return null; + if (windows) { + mvn = WebUtils.getAppHome().concat("/bin/mvnw.cmd"); + } else { + mvn = WebUtils.getAppHome().concat("/bin/mvnw"); + } } - File file = new File(setting); - AssertUtils.required( - !file.exists() || !file.isFile(), - String.format( - "Invalid maven-setting file path \"%s\", the path not exist or is not file", - setting)); - return setting; + return mvn.concat(mvnArgs); } private String getIllegalArgs(String param) { @@ -314,27 +278,19 @@ public String getMavenWorkHome() { @JsonIgnore public String getLog4BuildStart() { return String.format( - "%sproject : %s%nbranches: %s%ncommand : %s%n%n", - getLogHeader("maven install"), getName(), getBranches(), getMavenArgs()); + "%sproject : %s\nrefs: %s\ncommand : %s\n\n", + getLogHeader("maven install"), getName(), getRefs(), getMavenArgs()); } @JsonIgnore public String getLog4CloneStart() { return String.format( - "%sproject : %s%nbranches : %s%nworkspace: %s%n%n", - getLogHeader("git clone"), getName(), getBranches(), getAppSource()); + "%sproject : %s\nrefs : %s\nworkspace: %s\n\n", + getLogHeader("git clone"), getName(), getRefs(), getAppSource()); } @JsonIgnore private String getLogHeader(String header) { return "---------------------------------[ " + header + " ]---------------------------------\n"; } - - public boolean isHttpRepositoryUrl() { - return url != null && (url.trim().startsWith("https://") || url.trim().startsWith("http://")); - } - - public boolean isSshRepositoryUrl() { - return url != null && url.trim().startsWith("git@"); - } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java index 3726fb349e..08b46eb703 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java @@ -39,6 +39,8 @@ public interface ProjectService extends IService { */ RestResponse create(Project project); + boolean checkExists(Project project); + /** * Update the given Project * @@ -162,4 +164,6 @@ public interface ProjectService extends IService { * @return Check git */ GitAuthorizedErrorEnum gitCheck(Project project); + + List getAllTags(Project project); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java index 9c2516e618..9a1c9047b7 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java @@ -30,10 +30,9 @@ import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.streampark.console.base.exception.ApiDetailException; import org.apache.streampark.console.base.mybatis.pager.MybatisPager; -import org.apache.streampark.console.base.util.EncryptUtils; import org.apache.streampark.console.base.util.GZipUtils; import org.apache.streampark.console.base.util.GitUtils; -import org.apache.streampark.console.base.util.ShaHashUtils; +import org.apache.streampark.console.base.util.ObjectUtils; import org.apache.streampark.console.core.entity.Application; import org.apache.streampark.console.core.entity.Project; import org.apache.streampark.console.core.enums.BuildStateEnum; @@ -45,7 +44,6 @@ import org.apache.streampark.console.core.task.ProjectBuildTask; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.MemorySize; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; @@ -67,6 +65,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -100,77 +99,71 @@ public class ProjectServiceImpl extends ServiceImpl @Override public RestResponse create(Project project) { - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper().eq(Project::getName, - project.getName()); - long count = count(queryWrapper); RestResponse response = RestResponse.success(); - - ApiAlertException.throwIfTrue(count > 0, "project name already exists, add project failed"); - if (StringUtils.isNotBlank(project.getPassword())) { - String salt = ShaHashUtils.getRandomSalt(); - try { - String encrypt = EncryptUtils.encrypt(project.getPassword(), salt); - project.setSalt(salt); - project.setPassword(encrypt); - } catch (Exception e) { - log.error("Project password decrypt failed", e); - throw new ApiAlertException("Project github/gitlab password decrypt failed"); - } - } + project.setId(null); + ApiAlertException.throwIfTrue( + checkExists(project), "project name already exists, add project failed"); + Date date = new Date(); + project.setCreateTime(date); + project.setModifyTime(date); boolean status = save(project); - if (status) { return response.message("Add project successfully").data(true); + } else { + return response.message("Add project failed").data(false); + } + } + + @Override + public boolean checkExists(Project project) { + if (project.getId() != null) { + Project proj = getById(project.getId()); + if (proj != null && ObjectUtils.safeEquals(project.getName(), proj.getName())) { + return false; + } } - return response.message("Add project failed").data(false); + LambdaQueryWrapper queryWrapper = + new LambdaQueryWrapper() + .eq(Project::getName, project.getName()) + .eq(Project::getTeamId, project.getTeamId()); + return this.baseMapper.selectCount(queryWrapper) > 0; } @Override + @Transactional(rollbackFor = {Exception.class}) public boolean update(Project projectParam) { Project project = getById(projectParam.getId()); AssertUtils.notNull(project); ApiAlertException.throwIfFalse( project.getTeamId().equals(projectParam.getTeamId()), - "TeamId can't be changed, update project failed."); + "Team can't be changed, update project failed."); ApiAlertException.throwIfFalse( !project.getBuildState().equals(BuildStateEnum.BUILDING.get()), "The project is being built, update project failed."); - updateInternal(projectParam, project); - if (project.isHttpRepositoryUrl()) { - if (StringUtils.isBlank(projectParam.getUserName())) { - project.setUserName(null); - project.setPassword(null); - project.setSalt(null); - } else { - project.setUserName(projectParam.getUserName()); - if (!Objects.equals(projectParam.getPassword(), project.getPassword())) { - try { - String salt = ShaHashUtils.getRandomSalt(); - String encrypt = EncryptUtils.encrypt(projectParam.getPassword(), salt); - project.setPassword(encrypt); - project.setSalt(salt); - } catch (Exception e) { - log.error("The project github/gitlab password encrypt failed"); - throw new ApiAlertException(e); - } - } - } - } - if (project.isSshRepositoryUrl()) { + project.setName(projectParam.getName()); + project.setUrl(projectParam.getUrl()); + project.setRefs(projectParam.getRefs()); + project.setPrvkeyPath(projectParam.getPrvkeyPath()); + project.setUserName(projectParam.getUserName()); + project.setPassword(projectParam.getPassword()); + project.setPom(projectParam.getPom()); + project.setDescription(projectParam.getDescription()); + project.setBuildArgs(projectParam.getBuildArgs()); + project.setModifyTime(new Date()); + if (GitUtils.isSshRepositoryUrl(project.getUrl())) { project.setUserName(null); } else { project.setPrvkeyPath(null); } if (projectParam.getBuildState() != null) { project.setBuildState(projectParam.getBuildState()); - if (BuildStateEnum.NEED_REBUILD == BuildStateEnum.of(projectParam.getBuildState())) { + if (BuildStateEnum.of(projectParam.getBuildState()).equals(BuildStateEnum.NEED_REBUILD)) { List applications = listApps(project); // Update deployment status applications.forEach( (app) -> { log.info( - "update deploy by project: {}, appName:{}", project.getName(), - app.getJobName()); + "update deploy by project: {}, appName:{}", project.getName(), app.getJobName()); app.setRelease(ReleaseStateEnum.NEED_CHECK.get()); applicationManageService.updateRelease(app); }); @@ -180,18 +173,6 @@ public boolean update(Project projectParam) { return true; } - private static void updateInternal(Project projectParam, Project project) { - project.setName(projectParam.getName()); - project.setUrl(projectParam.getUrl()); - project.setBranches(projectParam.getBranches()); - project.setPrvkeyPath(projectParam.getPrvkeyPath()); - project.setUserName(projectParam.getUserName()); - project.setPassword(projectParam.getPassword()); - project.setPom(projectParam.getPom()); - project.setDescription(projectParam.getDescription()); - project.setBuildArgs(projectParam.getBuildArgs()); - } - @Override public boolean removeById(Long id) { Project project = getById(id); @@ -420,7 +401,12 @@ private String getBuildLogPath(Long projectId) { @Override public List getAllBranches(Project project) { try { - return GitUtils.getBranchList(remakeProject(project)); + GitUtils.GitGetRequest request = new GitUtils.GitGetRequest(); + request.setUrl(project.getUrl()); + request.setUsername(project.getUserName()); + request.setPassword(project.getPassword()); + request.setPrivateKey(project.getPrvkeyPath()); + return GitUtils.getBranches(request); } catch (Exception e) { throw new ApiDetailException(e); } @@ -429,7 +415,12 @@ public List getAllBranches(Project project) { @Override public GitAuthorizedErrorEnum gitCheck(Project project) { try { - GitUtils.getBranchList(remakeProject(project)); + GitUtils.GitGetRequest request = new GitUtils.GitGetRequest(); + request.setUrl(project.getUrl()); + request.setUsername(project.getUserName()); + request.setPassword(project.getPassword()); + request.setPrivateKey(project.getPrvkeyPath()); + GitUtils.getBranches(request); return GitAuthorizedErrorEnum.SUCCESS; } catch (Exception e) { String err = e.getMessage(); @@ -442,6 +433,20 @@ public GitAuthorizedErrorEnum gitCheck(Project project) { } } + @Override + public List getAllTags(Project project) { + try { + GitUtils.GitGetRequest request = new GitUtils.GitGetRequest(); + request.setUrl(project.getUrl()); + request.setUsername(project.getUserName()); + request.setPassword(project.getPassword()); + request.setPrivateKey(project.getPrvkeyPath()); + return GitUtils.getTags(request); + } catch (Exception e) { + throw new ApiDetailException(e); + } + } + private Project remakeProject(Project project) { if (Objects.nonNull(project.getId())) { return this.baseMapper.selectById(project.getId()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java index da511149e9..3571cbe27b 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java @@ -27,7 +27,6 @@ import ch.qos.logback.classic.Logger; import lombok.extern.slf4j.Slf4j; import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.InvalidRemoteException; import org.eclipse.jgit.lib.StoredConfig; import java.io.File; @@ -93,35 +92,32 @@ private boolean cloneSourceCode(Project project) { project.cleanCloned(); fileLogger.info("clone {}, {} starting...", project.getName(), project.getUrl()); fileLogger.info(project.getLog4CloneStart()); - Git git = GitUtils.clone(project); + + GitUtils.GitCloneRequest request = new GitUtils.GitCloneRequest(); + request.setUrl(project.getUrl()); + request.setRefs(project.getRefs()); + request.setStoreDir(project.getAppSource()); + request.setUsername(project.getUserName()); + request.setPassword(project.getPassword()); + request.setPrivateKey(project.getPrvkeyPath()); + + Git git = GitUtils.clone(request); StoredConfig config = git.getRepository().getConfig(); config.setBoolean("http", project.getUrl(), "sslVerify", false); config.setBoolean("https", project.getUrl(), "sslVerify", false); config.save(); File workTree = git.getRepository().getWorkTree(); printWorkTree(workTree, ""); - String successMsg = String.format("[StreamPark] project [%s] git clone successful!%n", project.getName()); + String successMsg = + String.format("[StreamPark] project [%s] git clone successful!\n", project.getName()); fileLogger.info(successMsg); git.close(); return true; } catch (Exception e) { - if (e instanceof InvalidRemoteException) { - if (project.isHttpRepositoryUrl()) { - String url = project - .getUrl() - .replaceAll( - "(https://|http://)(.*?)/(.*?)/(.*?)(\\.git|)\\s*$", - "git@$2:$3/$4.git"); - project.setUrl(url); - fileLogger.info( - "clone project by https(http) failed, Now try to clone project by ssh..."); - return cloneSourceCode(project); - } - } fileLogger.error( String.format( - "[StreamPark] project [%s] branch [%s] git clone failed, err: %s", - project.getName(), project.getBranches(), e)); + "[StreamPark] project [%s] refs [%s] git clone failed, err: %s", + project.getName(), project.getRefs(), e)); fileLogger.error(String.format("project %s clone error ", project.getName()), e); return false; } @@ -143,10 +139,11 @@ private void printWorkTree(File workTree, String space) { } private boolean projectBuild(Project project) { - int code = CommandUtils.execute( - project.getMavenWorkHome(), - Collections.singletonList(project.getMavenArgs()), - (line) -> fileLogger.info(line)); + int code = + CommandUtils.execute( + project.getMavenWorkHome(), + Collections.singletonList(project.getMavenArgs()), + (line) -> fileLogger.info(line)); return code == 0; } @@ -170,14 +167,15 @@ private void deploy(Project project) throws Exception { } // xzvf jar if (app.exists()) { - String cmd = String.format( - "tar -xzvf %s -C %s", app.getAbsolutePath(), deployPath.getAbsolutePath()); + String cmd = + String.format( + "tar -xzvf %s -C %s", app.getAbsolutePath(), deployPath.getAbsolutePath()); CommandUtils.execute(cmd); } } else { // 2) .jar file(normal or official standard flink project) Utils.requireCheckJarFile(app.toURI().toURL()); - String moduleName = app.getName().replace(Constants.JAR_SUFFIX, ""); + String moduleName = app.getName().replace(".jar", ""); File distHome = project.getDistHome(); File targetDir = new File(distHome, moduleName); if (!targetDir.exists()) { diff --git a/streampark-console/streampark-console-service/src/main/resources/db/schema-h2.sql b/streampark-console/streampark-console-service/src/main/resources/db/schema-h2.sql index 00c847d726..12844e01e9 100644 --- a/streampark-console/streampark-console-service/src/main/resources/db/schema-h2.sql +++ b/streampark-console/streampark-console-service/src/main/resources/db/schema-h2.sql @@ -165,25 +165,24 @@ create table if not exists `t_flink_log` ( -- Table structure for t_flink_project -- ---------------------------- create table if not exists `t_flink_project` ( - `id` bigint generated by default as identity not null, - `team_id` bigint not null, - `name` varchar(255) default null, - `url` varchar(255) default null, - `branches` varchar(64) default null, - `user_name` varchar(64) default null, - `password` varchar(64) default null, - `salt` varchar(26) default null, - `prvkey_path` varchar(128) default null, - `pom` varchar(255) default null, - `build_args` varchar(255) default null, - `type` tinyint default null, - `repository` tinyint default null, - `last_build` datetime default null, - `description` varchar(255) default null, - `build_state` tinyint default -1, - `create_time` datetime default null comment 'create time', - `modify_time` datetime default null comment 'modify time', - primary key(`id`) +`id` bigint generated by default as identity not null, +`team_id` bigint not null, +`name` varchar(255) default null, +`url` varchar(255) default null, +`refs` varchar(255) default null, +`user_name` varchar(64) default null, +`password` varchar(64) default null, +`prvkey_path` varchar(128) default null, +`pom` varchar(255) default null, +`build_args` varchar(255) default null, +`type` tinyint default null, +`repository` tinyint default null, +`last_build` datetime default null, +`description` varchar(255) default null, +`build_state` tinyint default -1, +`create_time` datetime default null comment 'create time', +`modify_time` datetime default null comment 'modify time', +primary key(`id`) ); diff --git a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml index 4a44e82c1b..ade3437a84 100644 --- a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml +++ b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml @@ -18,26 +18,6 @@ - - - - - - - - - - - - - - - - - - - -