diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..52ad9589387 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +.vscode +bin +build diff --git a/build_tcs.xml b/build_tcs.xml index 342606e3bea..8826db6f330 100644 --- a/build_tcs.xml +++ b/build_tcs.xml @@ -127,6 +127,16 @@ + + + + + + + + + + diff --git a/conf/com/topcoder/dde/util/DWLoad/RatingQubits.xml b/conf/com/topcoder/dde/util/DWLoad/RatingQubits.xml index 64429e6327b..090d55ba4ec 100755 --- a/conf/com/topcoder/dde/util/DWLoad/RatingQubits.xml +++ b/conf/com/topcoder/dde/util/DWLoad/RatingQubits.xml @@ -4,9 +4,18 @@ com.informix.jdbc.IfxDriver - jdbc:informix-sqli://63.118.154.190:1526/tcs_catalog:INFORMIXSERVER=devinformix10_shm;user=coder;password=altec + - 50 + 300 + + 150 + 149 + + + 06/01/2010 06:06:06 + NULL + + diff --git a/lib/jars/commons-logging-1.2.jar b/lib/jars/commons-logging-1.2.jar new file mode 100644 index 00000000000..93a3b9f6db4 Binary files /dev/null and b/lib/jars/commons-logging-1.2.jar differ diff --git a/lib/jars/fluent-hc-4.5.1.jar b/lib/jars/fluent-hc-4.5.1.jar new file mode 100644 index 00000000000..00c8d1c0a4d Binary files /dev/null and b/lib/jars/fluent-hc-4.5.1.jar differ diff --git a/lib/jars/httpclient-4.5.1.jar b/lib/jars/httpclient-4.5.1.jar new file mode 100644 index 00000000000..b9c0c1c2502 Binary files /dev/null and b/lib/jars/httpclient-4.5.1.jar differ diff --git a/lib/jars/httpcore-4.4.3.jar b/lib/jars/httpcore-4.4.3.jar new file mode 100644 index 00000000000..9f91ef663e2 Binary files /dev/null and b/lib/jars/httpcore-4.4.3.jar differ diff --git a/lib/jars/jackson-annotations-2.6.1.jar b/lib/jars/jackson-annotations-2.6.1.jar new file mode 100644 index 00000000000..fab18a67178 Binary files /dev/null and b/lib/jars/jackson-annotations-2.6.1.jar differ diff --git a/lib/jars/jackson-core-2.6.1.jar b/lib/jars/jackson-core-2.6.1.jar new file mode 100644 index 00000000000..acf508b186a Binary files /dev/null and b/lib/jars/jackson-core-2.6.1.jar differ diff --git a/lib/jars/jackson-databind-2.6.1.jar b/lib/jars/jackson-databind-2.6.1.jar new file mode 100644 index 00000000000..d7f35e25716 Binary files /dev/null and b/lib/jars/jackson-databind-2.6.1.jar differ diff --git a/scripts/dde/README.md b/scripts/dde/README.md new file mode 100644 index 00000000000..ab0300f04df --- /dev/null +++ b/scripts/dde/README.md @@ -0,0 +1,24 @@ +# Ratings calculation tool + +## Configuration +configuration is stored in `conf/com/topcoder/dde/RatingQubits.xml` and the following values should be updated: + +- `ConnectionURL` - should be updated to point to oltp database +- + + + +## Compilation + +Make sure your JAVA_HOME points to JDK 1.7 home directory and ANT_HOME points to ant 1.9.15 home directory. Then run the following + +`java -Dhttps.protocols=TLSv1.2 -cp %ANT_HOME%/lib/ant-launcher.jar org.apache.tools.ant.launch.Launcher compile -f build_tcs.xml` + +running ant manually is needed to allow it to download dependencies using https/ssl from maven repository and github + +## Running + +Update database configuration, move to `scripts/dde/` directory and run `runRatings.sh` script + +NOTE: if running on Windows, edit the $SEPARATOR variable in runRatings.sh and set it to `;` instead of `:` + diff --git a/scripts/dde/runRatings.sh b/scripts/dde/runRatings.sh index 19798a5b5f9..b7e2ff5907d 100755 --- a/scripts/dde/runRatings.sh +++ b/scripts/dde/runRatings.sh @@ -1,15 +1,27 @@ BASE=../.. +SEPARATOR=":" CP="" -CP=$CP:$BASE/conf -CP=$CP:$BASE/resources -CP=$CP:$BASE/lib/jars/log4j.jar -CP=$CP:$BASE/lib/jars/ifxjdbc.jar -CP=$CP:$BASE/lib/jars/jdbc7.0-1.2.jar -CP=$CP:$BASE/lib/jars/xml/xerces/xercesImpl.jar -CP=$CP:$BASE/build/classes -CP=$CP:$BASE/lib/jars/tcsUtil.jar -CP=$CP:$BASE/lib/jars/configmanager2.jar -CP=$CP:$BASE/lib/jars/jboss/xercesImpl.jar -CP=$CP:$BASE/lib/jars/jboss/xml-apis.jar +CP=$CP$SEPARATOR$BASE/conf +CP=$CP$SEPARATOR$BASE/resources +CP=$CP$SEPARATOR$BASE/lib/jars/log4j.jar +CP=$CP$SEPARATOR$BASE/lib/jars/ifxjdbc.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jdbc7.0-1.2.jar +CP=$CP$SEPARATOR$BASE/lib/jars/xml/xerces/xercesImpl.jar +CP=$CP$SEPARATOR$BASE/build/classes +CP=$CP$SEPARATOR$BASE/lib/jars/tcsUtil.jar +CP=$CP$SEPARATOR$BASE/lib/jars/configmanager2.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jboss/xercesImpl.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jboss/xml-apis.jar +CP=$CP$SEPARATOR$BASE/lib/jars/log4j-1.2.7.jar +CP=$CP$SEPARATOR$BASE/lib/bin/shared.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jackson-core-2.6.1.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jackson-databind-2.6.1.jar +CP=$CP$SEPARATOR$BASE/lib/jars/jackson-annotations-2.6.1.jar +CP=$CP$SEPARATOR$BASE/lib/jars/fluent-hc-4.5.1.jar +CP=$CP$SEPARATOR$BASE/lib/jars/httpclient-4.5.1.jar +CP=$CP$SEPARATOR$BASE/lib/jars/httpcore-4.4.3.jar +CP=$CP$SEPARATOR$BASE/lib/jars/commons-logging-1.2.jar +CP=$CP$SEPARATOR$BASE/lib/jars/ifxjdbc.jar -nohup java -cp $CP com.topcoder.dde.util.DWLoad.RatingQubits >> ./ratings.log 2>&1 & +echo $CP +java -Dhttps.protocols=TLSv1.2 -cp $CP com.topcoder.dde.util.DWLoad.RatingQubits diff --git a/src/main/com/topcoder/dde/util/DWLoad/RatingQubits.java b/src/main/com/topcoder/dde/util/DWLoad/RatingQubits.java index c34b308b44b..65b0d258755 100755 --- a/src/main/com/topcoder/dde/util/DWLoad/RatingQubits.java +++ b/src/main/com/topcoder/dde/util/DWLoad/RatingQubits.java @@ -126,6 +126,11 @@ public class RatingQubits { " and p.project_id in (select pi.project_id from project_info pi where pi.project_info_type_id=13 " + "and (pi.value='Yes' or pi.value='yes')) "; + /** + * Maximum number of groups to fetch from the api + */ + private static final int MAX_GROUPS = 1000; + /** * Load phase ids and cutoffs configurations from file * @param args the command arguments @@ -154,6 +159,16 @@ public static void main(String[] args) { } } + Map> groups = new HashMap>(); + try{ + log.info("fetching groups"); + groups = Util.getGroups(0, MAX_GROUPS); + log.info("got groups"); + } catch (Exception ex){ + log.error("cant fetch groups", ex); + System.exit(1); + } + String jdbcDriver; String connectionURL; Map projects = new HashMap(); @@ -217,7 +232,7 @@ public static void main(String[] args) { c = DriverManager.getConnection(connectionURL); c.setAutoCommit(true); - tmp.runAllScores(c, historyLength, projects); + tmp.runAllScores(c, historyLength, projects, groups); } catch (Exception e) { e.printStackTrace(); } finally { @@ -236,7 +251,7 @@ public static void main(String[] args) { * @param projects the projects map with keyis phase id and value is cut off date * @since 1.3 */ - public void runAllScores(Connection conn, String historyLength, Map projects) { + public void runAllScores(Connection conn, String historyLength, Map projects, Map> groups) { Date newPhasesCutoff = null; try { @@ -262,14 +277,32 @@ public void runAllScores(Connection conn, String historyLength, Map projects, Date newPhasesCutoff, Date uiPrototypeCutoff, Date riaBuildCutoff, String group, String childGroups){ + + log.info("running scores for group: "+group); + for (Map.Entry entry : projects.entrySet()) { - runScore(conn, historyLength, entry.getKey(), entry.getValue()); + runScore(conn, historyLength, entry.getKey(), entry.getValue(), group, childGroups); } // phase ids without cut off int[] phaseIds1 = new int [] { DESIGN_PHASE_ID, DEV_PHASE_ID } ; for (int i = 0; i < phaseIds1.length; i++) { if(!projects.containsKey(phaseIds1[i])) { - runScore(conn, historyLength, phaseIds1[i]); + runScore(conn, historyLength, phaseIds1[i], group, childGroups); } } @@ -278,26 +311,26 @@ public void runAllScores(Connection conn, String historyLength, Map oldRatingsMap = getOldRatingsMap(conn, categoryId); - HashMap newRatingsMap = getNewRatingsMap(conn, categoryId); + HashMap oldRatingsMap = getOldRatingsMap(conn, categoryId, groupId); + HashMap newRatingsMap = getNewRatingsMap(conn, categoryId, groupId); while (rs.next()) { @@ -567,10 +610,6 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his rating r = (rating) ratings.get("" + coder); - //update project_result record with new and old rating - sqlStr.replace(0, sqlStr.length(), "UPDATE project_result SET old_rating = ?, new_rating = ? "); - sqlStr.append(" WHERE project_id = ? and user_id = ? "); - key = rs.getInt("project_id") + "-" + coder; boolean doit = false; if (r.num_ratings == 0) { @@ -591,19 +630,64 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his //only update the db if something actually changed if (doit) { - ps = conn.prepareStatement(sqlStr.toString()); - if (r.num_ratings == 0) { - ps.setNull(1, Types.DOUBLE); + if (groupId==null){ + //update project_result record with new and old rating + sqlStr.replace(0, sqlStr.length(), "UPDATE project_result SET old_rating = ?, new_rating = ? "); + sqlStr.append(" WHERE project_id = ? and user_id = ? "); + ps = conn.prepareStatement(sqlStr.toString()); + if (r.num_ratings == 0) { + ps.setNull(1, Types.DOUBLE); + } else { + ps.setDouble(1, r.rating); + } + ps.setInt(2, newrating); + ps.setInt(3, rs.getInt("project_id")); + ps.setInt(4, coder); + + ps.execute(); + ps.close(); + ps = null; } else { - ps.setDouble(1, r.rating); + //update project_result record with new and old rating + sqlStr.replace(0, sqlStr.length(), "UPDATE group_project_result SET old_rating = ?, new_rating = ? "); + sqlStr.append(" WHERE project_id = ? and user_id = ? and group_id = ?"); + ps = conn.prepareStatement(sqlStr.toString()); + if (r.num_ratings == 0) { + ps.setNull(1, Types.DOUBLE); + } else { + ps.setDouble(1, r.rating); + } + ps.setInt(2, newrating); + ps.setInt(3, rs.getInt("project_id")); + ps.setInt(4, coder); + ps.setInt(5, new Integer(groupId)); + + int ret = ps.executeUpdate(); + ps.close(); + ps = null; + + if (ret==0){ + sqlStr.replace(0, sqlStr.length(), "INSERT INTO group_project_result(project_id, user_id, group_id, old_rating, new_rating)"); + sqlStr.append(" values (?,?,?,?,?)"); + ps = conn.prepareStatement(sqlStr.toString()); + if (r.num_ratings == 0) { + ps.setNull(4, Types.DOUBLE); + } else { + ps.setDouble(4, r.rating); + } + ps.setInt(5, newrating); + + ps.setInt(1, rs.getInt("project_id")); + ps.setInt(2, coder); + ps.setInt(3, new Integer(groupId)); + + ps.execute(); + + ps.close(); + ps = null; + } + } - ps.setInt(2, newrating); - ps.setInt(3, rs.getInt("project_id")); - ps.setInt(4, coder); - - ps.execute(); - ps.close(); - ps = null; } //update user_rating @@ -635,9 +719,15 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his rating r = (rating) vals[i]; //System.out.println(r.user_id + "\t" + r.rating); - - sqlStr.replace(0, sqlStr.length(), "UPDATE user_rating set rating = ?, vol = ?, rating_no_vol = ?, last_rated_project_id = ?, num_ratings = ? "); + String table = "user_rating"; + if (groupId!=null){ + table = "group_user_rating"; + } + sqlStr.replace(0, sqlStr.length(), "UPDATE "+table+" set rating = ?, vol = ?, rating_no_vol = ?, last_rated_project_id = ?, num_ratings = ? "); sqlStr.append(" where phase_id = ? and user_id = ?"); + if (groupId!=null){ + sqlStr.append(" and group_id = ? "); + } ps = conn.prepareStatement(sqlStr.toString()); ps.setDouble(1, r.rating); @@ -647,6 +737,9 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his ps.setInt(5, r.num_ratings); ps.setInt(6, phaseId); ps.setDouble(7, r.user_id); + if (groupId!=null){ + ps.setInt(8, new Integer(groupId)); + } int retVal = ps.executeUpdate(); @@ -655,8 +748,16 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his if (retVal == 0) { - sqlStr.replace(0, sqlStr.length(), "INSERT INTO user_rating (user_id, phase_id, rating, vol, rating_no_vol, last_rated_project_id, num_ratings) "); - sqlStr.append(" values (?, ?, ?, ?, ?, ?, ?)"); + sqlStr.replace(0, sqlStr.length(), "INSERT INTO "+table+" (user_id, phase_id, rating, vol, rating_no_vol, last_rated_project_id, num_ratings"); + if (groupId!=null){ + sqlStr.append(", group_id"); + } + sqlStr.append(") values (?, ?, ?, ?, ?, ?, ?"); + if (groupId!=null){ + sqlStr.append(", ?"); + } + sqlStr.append(")"); + ps = conn.prepareStatement(sqlStr.toString()); ps.setDouble(1, r.user_id); ps.setInt(2, phaseId); @@ -665,6 +766,9 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his ps.setDouble(5, r.rating_no_vol); ps.setInt(6, r.last_project_rated); ps.setInt(7, r.num_ratings); + if (groupId!=null){ + ps.setInt(8, new Integer(groupId)); + } ps.execute(); @@ -703,20 +807,37 @@ private void rateProjects(Connection conn, ResultSet rs, int phaseId, String his RATED_CONSTRAINTS_SQL_FRAGMENT + "and pr.final_score is not null "; + /** + * SQL query to retrieve old ratings for group projects + */ + private final String OLD_RATINGS_GROUP = "select gpr.project_id AS project_id, gpr.user_id AS user_id, pr.old_rating AS old_rating from group_project_result gpr, project_result pr, project p " + + "where p.project_id = pr.project_id " + + "and p.project_id = gpr.project_id " + + "and p.project_status_id in " + NEW_RATING_STATUSES + " " + + "and p.project_category_id = ? " + + "and gpr.rating_ind =1 " + + "and gpr.group_id = ?" + + RATED_CONSTRAINTS_SQL_FRAGMENT + + "and pr.final_score is not null "; + /** * return a mapping between project id/user id and what's currently in the database for old rating * the key will be - * * @return */ - private HashMap getOldRatingsMap(Connection conn, int categoryId) throws SQLException { + private HashMap getOldRatingsMap(Connection conn, int categoryId, String groupId) throws SQLException { HashMap ret = new HashMap(); PreparedStatement ps = null; ResultSet rs = null; try { - ps = conn.prepareStatement(OLD_RATINGS); + String sql = groupId==null?OLD_RATINGS:OLD_RATINGS_GROUP; + ps = conn.prepareStatement(sql); ps.setInt(1, categoryId); + if (groupId!=null){ + ps.setInt(2, new Integer(groupId)); + } rs = ps.executeQuery(); while (rs.next()) { ret.put(rs.getString("project_id") + "-" + rs.getString("user_id"), rs.getString("old_rating") == null ? null : rs.getInt("old_rating")); @@ -741,20 +862,37 @@ private HashMap getOldRatingsMap(Connection conn, int categoryI RATED_CONSTRAINTS_SQL_FRAGMENT + "and pr.final_score is not null "; + /** + * SQL query to retrieve new ratings + */ + private final String NEW_RATINGS_GROUP = "select gpr.project_id, gpr.user_id, gpr.new_rating from group_project_result gpr, project_result pr , project p " + + "where p.project_id = pr.project_id " + + "and p.project_id = gpr.project_id " + + "and p.project_status_id in " + NEW_RATING_STATUSES + " " + + "and p.project_category_id = ? " + + "and pr.rating_ind =1 " + + "and gpr.group_id = ? " + + RATED_CONSTRAINTS_SQL_FRAGMENT + + "and pr.final_score is not null "; + /** * return a mapping between project id/user id and what's currently in the database for new rating * the key will be - * * @return */ - private HashMap getNewRatingsMap(Connection conn, int categoryId) throws SQLException { + private HashMap getNewRatingsMap(Connection conn, int categoryId, String groupId) throws SQLException { HashMap ret = new HashMap(); PreparedStatement ps = null; ResultSet rs = null; try { - ps = conn.prepareStatement(NEW_RATINGS); + String sql = groupId == null ? NEW_RATINGS : NEW_RATINGS_GROUP; + ps = conn.prepareStatement(sql); ps.setInt(1, categoryId); + if (groupId!=null){ + ps.setInt(2, new Integer(groupId)); + } rs = ps.executeQuery(); while (rs.next()) { ret.put(rs.getString("project_id") + "-" + rs.getString("user_id"), rs.getString("new_rating") == null ? null : rs.getInt("new_rating")); @@ -767,7 +905,7 @@ private HashMap getNewRatingsMap(Connection conn, int categoryI } - private void updateRatingOrder(Connection conn, int phase) throws Exception { + private void updateRatingOrder(Connection conn, int phase, String groupId) throws Exception { // update rating_order column log.info("UPDATING rating_order COLUMN.... for phase " + phase); ResultSet rs2 = null; @@ -775,22 +913,37 @@ private void updateRatingOrder(Connection conn, int phase) throws Exception { PreparedStatement psUpd = null; try { StringBuffer sqlStr = new StringBuffer(300); + String table = "project_result"; + if (groupId!=null){ + table = "group_project_result"; + } sqlStr.append("select user_id, project_category_id, p.project_id, rating_order "); sqlStr.append(" ,substr(pi.value, 1, 2) as month, substr(pi.value, 4, 2) as day, substr(pi.value, 7, 4) as year").append( ", case when substr(pi.value, 18,2)='PM' then round(substr(pi.value, 12, 2)) +12 else round(substr(pi.value, 12, 2)) end as hour "); - sqlStr.append("from project_result pr, project_info pi, project p "); + sqlStr.append("from "+table+" pr, project_info pi, project p "); sqlStr.append("where pi.project_info_type_id =22 "); sqlStr.append("and pi.project_id = pr.project_id "); sqlStr.append("and pi.project_id = p.project_id "); sqlStr.append("and p.project_category_id = ? "); sqlStr.append("and p.project_status_id in (4,5,6, 7) "); + if (groupId!=null){ + sqlStr.append("and pr.group_id = ? "); + } sqlStr.append("order by user_id, year, month, day,hour, p.project_id "); ps = conn.prepareStatement(sqlStr.toString()); ps.setInt(1, phase - 111); + if (groupId!=null){ + ps.setInt(2, new Integer(groupId)); + } rs2 = ps.executeQuery(); - psUpd = conn.prepareStatement("UPDATE project_result SET rating_order=? where user_id=? and project_id=?"); + sqlStr.replace(0,sqlStr.length(),"UPDATE "+table+" SET rating_order=? where user_id=? and project_id=?"); + if (groupId!=null){ + sqlStr.append(" and group_id = ?"); + } + psUpd = conn.prepareStatement(sqlStr.toString()); + long prevUser = -1; int ratingOrder = 1; @@ -806,6 +959,9 @@ private void updateRatingOrder(Connection conn, int phase) throws Exception { psUpd.setInt(1, ratingOrder); psUpd.setLong(2, rs2.getLong("user_id")); psUpd.setLong(3, rs2.getLong("project_id")); + if (groupId!=null){ + psUpd.setInt(4,new Integer(groupId)); + } int retVal = psUpd.executeUpdate(); if (retVal != 1) { diff --git a/src/main/com/topcoder/dde/util/DWLoad/Util.java b/src/main/com/topcoder/dde/util/DWLoad/Util.java new file mode 100644 index 00000000000..565a6cbaec6 --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/Util.java @@ -0,0 +1,187 @@ +package com.topcoder.dde.util.DWLoad; + +import com.topcoder.dde.util.DWLoad.entity.*; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.client.fluent.Executor; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.config.Registry; +import org.apache.http.ssl.SSLContexts; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.client.HttpClient; + + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import javax.net.ssl.SSLContext; + +import com.topcoder.shared.util.logging.Logger; + + +import java.io.IOException; +import java.util.*; + +public class Util { + + private static String groupsApiBase = "https://api.topcoder-dev.com/v5/groups/"; + private static final Logger logger = Logger.getLogger(Util.class); + + + + /** + * Creates mapping of groups ids to list of child groups (multiple hierarchy levels are supported + * @return group mapping + * @throws IOException - if there is error getting group details from the api + */ + public static Map> getGroups(int offset, int total) throws Exception { + // get groups list + logger.info("start fetching groups"); + String token = doGetToken(); + + + if (System.getenv("TC_GROUPS_API")!=null){ + groupsApiBase = System.getenv("TC_GROUPS_API"); + } + + PoolingHttpClientConnectionManager clientConnectionManager = new PoolingHttpClientConnectionManager(getRegistry()); + HttpClient client = HttpClients.custom().setConnectionManager(clientConnectionManager).build(); + Executor executor = Executor.newInstance(client); + + String url = groupsApiBase + "?page=" + (1 + offset/total) + "&perPage="+total+"&includeSubGroups=true&oneLevel=false"; + String groupsResponse = executor.execute(Request.Get(url) + .addHeader("Authorization", "Bearer "+token)) + .returnContent().asString(); + ObjectMapper mapper = new ObjectMapper(); + List result = mapper.readValue(groupsResponse, new TypeReference>(){}); + Map> groupChildren = new TreeMap>(); + for (GroupDetail gd : result){ + if (gd.getOldId()==null){ + logger.warn("Group has oldId set as null and will be ignored: "+gd.getId()); + continue; + } + Set children = new HashSet(); + if (gd.getSubGroups()!=null) { + for (GroupDetail cgd : gd.getSubGroups()) { + if (cgd.getOldId() == null) { + logger.warn("Group has oldId set as null: " + cgd.getId()); + continue; + } + children.add(cgd.getOldId()); + } + } else { +// logger.warn("Child groups are null for: "+gd.getId()+" "+gd.getName()); + } + groupChildren.put(gd.getOldId(),children); + + logger.info("Child groups for " + gd.getOldId() + ":" + Util.join(",", groupChildren.get(gd.getOldId()))); + } + logger.info("Groups fetched: "+groupChildren.size()); + return groupChildren; + } + + private static String doGetToken() throws Exception { + String clientId = System.getenv("TC_CLIENT_ID"); + String clientSecret = System.getenv("TC_CLIENT_SECRET"); + String audience = System.getenv("TC_CLIENT_AUDIENCE"); + String domain = System.getenv("TC_CLIENT_DOMAIN"); + String auth0ProxyServerUrl = System.getenv("TC_CLIENT_PROXY_URL"); + return getMachineToken(clientId,clientSecret,audience,domain,auth0ProxyServerUrl); + } + + //helper method to join String in Java 7 + public static String join(String delimeter, Set set) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String item : set) { + if (first) { + first = false; + } else { + sb.append(delimeter); + } + sb.append(item); + } + return sb.toString(); + } + + private static String getMachineToken(String clientId, String clientSecret, String audience, String domain, String auth0ProxyServerUrl) throws Exception { + if (audience == null || audience.trim().length() == 0) { + throw new IllegalArgumentException("The audience should be non-null and non-empty string"); + } + + if (domain == null || domain.trim().length() == 0) { + throw new IllegalArgumentException("The domain should be non-null and non-empty string"); + } + + + String auth0Url = "https://" + domain + "/token" ; + String authServerUrl = auth0ProxyServerUrl != null && auth0ProxyServerUrl.trim().length() > 0 ? auth0ProxyServerUrl : auth0Url; + + Map postMap = new HashMap(); + postMap.put("client_id", clientId); + postMap.put("client_secret", clientSecret); + postMap.put("grant_type", "client_credentials"); + postMap.put("audience", audience); + postMap.put("auth0_url", auth0Url); + postMap.put("fresh_token", "false"); + ObjectMapper mapper = new ObjectMapper(); + logger.info("auth url "+authServerUrl); + String response = Request.Post(authServerUrl).bodyString(mapper.writeValueAsString(postMap), ContentType.APPLICATION_JSON) + .execute().returnContent().asString(); + String token = mapper.readTree(response).path("access_token").asText(); + logger.info("got the m2m token, length "+token.length()); + return token; + } + + public static Map> getGroup(String singleGroupId) throws Exception { + // get groups list + logger.info("start fetching groups"); + String token = Util.doGetToken(); + + + if (System.getenv("TC_GROUPS_API")!=null){ + groupsApiBase = System.getenv("TC_GROUPS_API"); + } + + String url = groupsApiBase + "?page=" + singleGroupId; + String groupsResponse = Request.Get(url) + .addHeader("Authorization", "Bearer "+token) + .execute().returnContent().asString(); + ObjectMapper mapper = new ObjectMapper(); + GroupDetail result = mapper.readValue(groupsResponse, GroupDetail.class); + Set children = new HashSet(); + for (GroupDetail cgd : result.getSubGroups()){ + if (cgd.getOldId()==null){ + logger.warn("Group has oldId set as null: "+cgd.getId()); + continue; + } + children.add(cgd.getOldId()); + } + logger.info("Child groups for " + result.getOldId() + ":" + Util.join(",", children)); + + Map> results = new HashMap>(); + if (result.getOldId()==null){ + logger.warn("Group has oldId set as null and will be ignored: "+result.getId()); + } else { + results.put(result.getOldId(),children); + } + + return results; + } + + private static Registry getRegistry() throws KeyManagementException, NoSuchAlgorithmException { + SSLContext sslContext = SSLContexts.custom().build(); + SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, + new String[]{"TLSv1.2"}, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); + return RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .register("https", sslConnectionSocketFactory) + .build(); + } +} diff --git a/src/main/com/topcoder/dde/util/DWLoad/entity/GroupDetail.java b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupDetail.java new file mode 100644 index 00000000000..37a8a98b992 --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupDetail.java @@ -0,0 +1,270 @@ +package com.topcoder.dde.util.DWLoad.entity; + +import java.util.List; + +public class GroupDetail { + private String id; + private String oldId; + private String ssoId; + private String createdAt; + private String createdBy; + private String updatedBy; + private String updatedAt; + private boolean selfRegister; + private String domain; + private String name; + private String description; + private boolean privateGroup; + + public GroupDetail(String id, String oldId, String ssoId, String createdAt, String createdBy, String updatedBy, String updatedAt, boolean selfRegister, String domain, String name, String description, boolean privateGroup, String status, List subGroups, List parentGroups) { + this.id = id; + this.oldId = oldId; + this.ssoId = ssoId; + this.createdAt = createdAt; + this.createdBy = createdBy; + this.updatedBy = updatedBy; + this.updatedAt = updatedAt; + this.selfRegister = selfRegister; + this.domain = domain; + this.name = name; + this.description = description; + this.privateGroup = privateGroup; + this.status = status; + this.subGroups = subGroups; + this.parentGroups = parentGroups; + } + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + public String getOldId() { + return this.oldId; + } + + public void setOldId(String oldId) { + this.oldId = oldId; + } + + public String getSsoId() { + return this.ssoId; + } + + public void setSsoId(String ssoId) { + this.ssoId = ssoId; + } + + public String getCreatedAt() { + return this.createdAt; + } + + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + public String getCreatedBy() { + return this.createdBy; + } + + public void setCreatedBy(String createdBy) { + this.createdBy = createdBy; + } + + public String getUpdatedBy() { + return this.updatedBy; + } + + public void setUpdatedBy(String updatedBy) { + this.updatedBy = updatedBy; + } + + public String getUpdatedAt() { + return this.updatedAt; + } + + public void setUpdatedAt(String updatedAt) { + this.updatedAt = updatedAt; + } + + public boolean isSelfRegister() { + return this.selfRegister; + } + + public boolean getSelfRegister() { + return this.selfRegister; + } + + public void setSelfRegister(boolean selfRegister) { + this.selfRegister = selfRegister; + } + + public String getDomain() { + return this.domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return this.description; + } + + public void setDescription(String description) { + this.description = description; + } + + public boolean isPrivateGroup() { + return this.privateGroup; + } + + public boolean getPrivateGroup() { + return this.privateGroup; + } + + public void setPrivateGroup(boolean privateGroup) { + this.privateGroup = privateGroup; + } + + public String getStatus() { + return this.status; + } + + public void setStatus(String status) { + this.status = status; + } + + public List getSubGroups() { + return this.subGroups; + } + + public void setSubGroups(List subGroups) { + this.subGroups = subGroups; + } + + public List getParentGroups() { + return this.parentGroups; + } + + public void setParentGroups(List parentGroups) { + this.parentGroups = parentGroups; + } + + public GroupDetail id(String id) { + this.id = id; + return this; + } + + public GroupDetail oldId(String oldId) { + this.oldId = oldId; + return this; + } + + public GroupDetail ssoId(String ssoId) { + this.ssoId = ssoId; + return this; + } + + public GroupDetail createdAt(String createdAt) { + this.createdAt = createdAt; + return this; + } + + public GroupDetail createdBy(String createdBy) { + this.createdBy = createdBy; + return this; + } + + public GroupDetail updatedBy(String updatedBy) { + this.updatedBy = updatedBy; + return this; + } + + public GroupDetail updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return this; + } + + public GroupDetail selfRegister(boolean selfRegister) { + this.selfRegister = selfRegister; + return this; + } + + public GroupDetail domain(String domain) { + this.domain = domain; + return this; + } + + public GroupDetail name(String name) { + this.name = name; + return this; + } + + public GroupDetail description(String description) { + this.description = description; + return this; + } + + public GroupDetail privateGroup(boolean privateGroup) { + this.privateGroup = privateGroup; + return this; + } + + public GroupDetail status(String status) { + this.status = status; + return this; + } + + public GroupDetail subGroups(List subGroups) { + this.subGroups = subGroups; + return this; + } + + public GroupDetail parentGroups(List parentGroups) { + this.parentGroups = parentGroups; + return this; + } + + @Override + public String toString() { + return "{" + + " id='" + getId() + "'" + + ", oldId='" + getOldId() + "'" + + ", ssoId='" + getSsoId() + "'" + + ", createdAt='" + getCreatedAt() + "'" + + ", createdBy='" + getCreatedBy() + "'" + + ", updatedBy='" + getUpdatedBy() + "'" + + ", updatedAt='" + getUpdatedAt() + "'" + + ", selfRegister='" + isSelfRegister() + "'" + + ", domain='" + getDomain() + "'" + + ", name='" + getName() + "'" + + ", description='" + getDescription() + "'" + + ", privateGroup='" + isPrivateGroup() + "'" + + ", status='" + getStatus() + "'" + + ", subGroups='" + getSubGroups() + "'" + + ", parentGroups='" + getParentGroups() + "'" + + "}"; + } + private String status; + + private List subGroups; + private List parentGroups; + + public GroupDetail(){ + + } + + +} diff --git a/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResponse.java b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResponse.java new file mode 100644 index 00000000000..542b1b8acaa --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResponse.java @@ -0,0 +1,57 @@ +package com.topcoder.dde.util.DWLoad.entity; + +public class GroupResponse { + private String id; + private GroupResult result; + private String version; + + public GroupResponse(){ + + } + + public GroupResponse(String id, GroupResult result, String version) { + this.id = id; + this.result = result; + this.version = version; + } + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + public GroupResult getResult() { + return this.result; + } + + public void setResult(GroupResult result) { + this.result = result; + } + + public String getVersion() { + return this.version; + } + + public void setVersion(String version) { + this.version = version; + } + + public GroupResponse id(String id) { + this.id = id; + return this; + } + + public GroupResponse result(GroupResult result) { + this.result = result; + return this; + } + + public GroupResponse version(String version) { + this.version = version; + return this; + } + +} \ No newline at end of file diff --git a/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResult.java b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResult.java new file mode 100644 index 00000000000..3e2400a5f4c --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupResult.java @@ -0,0 +1,76 @@ +package com.topcoder.dde.util.DWLoad.entity; + +public class GroupResult { + private boolean success; + private int status; + private String metadata; + private GroupDetail content; + + public GroupResult(){ + + } + + public GroupResult(boolean success, int status, String metadata, GroupDetail content) { + this.success = success; + this.status = status; + this.metadata = metadata; + this.content = content; + } + + public boolean isSuccess() { + return this.success; + } + + public boolean getSuccess() { + return this.success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public int getStatus() { + return this.status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMetadata() { + return this.metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } + + public GroupDetail getContent() { + return this.content; + } + + public void setContent(GroupDetail content) { + this.content = content; + } + + public GroupResult success(boolean success) { + this.success = success; + return this; + } + + public GroupResult status(int status) { + this.status = status; + return this; + } + + public GroupResult metadata(String metadata) { + this.metadata = metadata; + return this; + } + + public GroupResult content(GroupDetail content) { + this.content = content; + return this; + } + +} diff --git a/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResponse.java b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResponse.java new file mode 100644 index 00000000000..ee38b46e196 --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResponse.java @@ -0,0 +1,57 @@ +package com.topcoder.dde.util.DWLoad.entity; + + +public class GroupsResponse { + private String id; + private GroupsResult result; + private String version; + + public GroupsResponse(){ + + } + + public GroupsResponse(String id, GroupsResult result, String version) { + this.id = id; + this.result = result; + this.version = version; + } + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + public GroupsResult getResult() { + return this.result; + } + + public void setResult(GroupsResult result) { + this.result = result; + } + + public String getVersion() { + return this.version; + } + + public void setVersion(String version) { + this.version = version; + } + + public GroupsResponse id(String id) { + this.id = id; + return this; + } + + public GroupsResponse result(GroupsResult result) { + this.result = result; + return this; + } + + public GroupsResponse version(String version) { + this.version = version; + return this; + } +} \ No newline at end of file diff --git a/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResult.java b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResult.java new file mode 100644 index 00000000000..bd56d6019ae --- /dev/null +++ b/src/main/com/topcoder/dde/util/DWLoad/entity/GroupsResult.java @@ -0,0 +1,77 @@ +package com.topcoder.dde.util.DWLoad.entity; + +import java.util.List; + +public class GroupsResult { + private boolean success; + private int status; + private String metadata; + private List content; + + public GroupsResult(){ + + } + + public GroupsResult(boolean success, int status, String metadata, List content) { + this.success = success; + this.status = status; + this.metadata = metadata; + this.content = content; + } + + public boolean isSuccess() { + return this.success; + } + + public boolean getSuccess() { + return this.success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public int getStatus() { + return this.status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMetadata() { + return this.metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } + + public List getContent() { + return this.content; + } + + public void setContent(List content) { + this.content = content; + } + + public GroupsResult success(boolean success) { + this.success = success; + return this; + } + + public GroupsResult status(int status) { + this.status = status; + return this; + } + + public GroupsResult metadata(String metadata) { + this.metadata = metadata; + return this; + } + + public GroupsResult content(List content) { + this.content = content; + return this; + } +} \ No newline at end of file