Skip to content

Commit d2c161e

Browse files
phasenraum2010phasenraum2010
committed
working on #141 #142 #143 
1 parent 7216c06 commit d2c161e

File tree

9 files changed

+88
-25
lines changed

9 files changed

+88
-25
lines changed

‎src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public String fetchTweetsFromTwitterSearchRequest(Model model) {
6262
String subtitle = "/scheduled/tweets/fetch";
6363
String symbol = Symbols.TASK.toString();
6464
model = controllerHelper.setupPage(model,title,subtitle,symbol);
65-
mqAsyncStartTask.fetchUsersFromDefinedUserList();
65+
mqAsyncStartTask.fetchTweetsFromTwitterSearch();
6666
return "scheduled/taskStarted";
6767
}
6868

‎src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.woehlke.twitterwall.oodm.repositories;
22

3+
import org.springframework.data.domain.Page;
4+
import org.springframework.data.domain.Pageable;
35
import org.springframework.stereotype.Repository;
46
import org.woehlke.twitterwall.oodm.entities.Mention;
57
import org.woehlke.twitterwall.oodm.repositories.common.DomainRepository;
@@ -17,4 +19,5 @@ public interface MentionRepository extends DomainRepository<Mention>,MentionRepo
1719

1820
Mention findByScreenName(String screenName);
1921

22+
Page<Mention> findAllByUserNull(Pageable pageRequest);
2023
}

‎src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.woehlke.twitterwall.oodm.service;
22

3+
import org.springframework.data.domain.Page;
4+
import org.springframework.data.domain.Pageable;
35
import org.woehlke.twitterwall.oodm.entities.Task;
46
import org.woehlke.twitterwall.oodm.entities.Mention;
57
import org.woehlke.twitterwall.oodm.service.common.DomainServiceWithIdTwitter;
@@ -13,4 +15,6 @@
1315
public interface MentionService extends DomainServiceWithScreenName<Mention>,DomainServiceWithIdTwitter<Mention>,DomainServiceWithTask<Mention> {
1416

1517
Mention createProxyMention(Mention mention, Task task);
18+
19+
Page<Mention> getAllWithoutPersistentUser(Pageable pageRequest);
1620
}

‎src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,9 @@ public Mention createProxyMention(Mention mention, Task task) {
6565
mention = mentionRepository.save(mention);
6666
return mention;
6767
}
68+
69+
@Override
70+
public Page<Mention> getAllWithoutPersistentUser(Pageable pageRequest) {
71+
return mentionRepository.findAllByUserNull(pageRequest);
72+
}
6873
}

‎src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ public void fetchUsersFromDefinedUserList() {
5151
}
5252

5353
private void send(TaskType taskType){
54-
String msg = "Start task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER;
54+
String msg = "START Task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER;
55+
log.info(msg);
5556
CountedEntities countedEntities = countedEntitiesService.countAll();
5657
Task task = taskService.create(msg, taskType,countedEntities);
5758
TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted());
@@ -61,7 +62,6 @@ private void send(TaskType taskType){
6162
.setHeader("task_type", task.getTaskType())
6263
.build();
6364
MessagingTemplate mqTemplate = new MessagingTemplate();
64-
log.info(msg);
6565
mqTemplate.send(executorChannelForAsyncStart, mqMessage);
6666
}
6767

‎src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,19 +55,25 @@ public List<TweetFromTwitter> splitMessage(Message<TaskMessage> message) {
5555
List<Long> worklistTwitterIds = new ArrayList<>();
5656
boolean hasNext=true;
5757
Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize());
58+
int lfdNr = 0;
59+
int all = 0;
5860
while(hasNext) {
5961
Page<org.woehlke.twitterwall.oodm.entities.Tweet> tweetTwitterIds = tweetService.getAll(pageRequest);
6062
for(org.woehlke.twitterwall.oodm.entities.Tweet tweetTwitterId:tweetTwitterIds.getContent()){
61-
log.debug("### tweetService.findAllTwitterIds (1): "+tweetTwitterId.getIdTwitter());
63+
lfdNr++;
64+
all++;
65+
log.debug("### tweetService.findAllTwitterIds from DB ("+lfdNr+"): "+tweetTwitterId.getIdTwitter());
6266
worklistTwitterIds.add(tweetTwitterId.getIdTwitter());
6367
}
6468
hasNext = tweetTwitterIds.hasNext();
6569
pageRequest = pageRequest.next();
6670
}
6771
int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls();
6872
List<TweetFromTwitter> tweets = new ArrayList<>();
73+
lfdNr = 0;
6974
for(Long tweetTwitterId : worklistTwitterIds){
70-
log.debug("### tweetService.findAllTwitterIds (2): "+tweetTwitterId);
75+
lfdNr++;
76+
log.debug("### witterApiService.findOneTweetById from Twiiter API ("+lfdNr+" of "+all+"): "+tweetTwitterId);
7177
Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId);
7278
TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter);
7379
tweets.add(result);

‎src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,30 +59,34 @@ public List<TwitterProfileMessage> splitMessage(Message<TaskMessage> message) {
5959
Task task = taskService.findById(id);
6060
task = taskService.start(task,countedEntities);
6161
List<String> screenNames = new ArrayList<>();
62-
int allLoop = 0;
63-
int loopId = 0;
62+
int lfdNr = 0;
63+
int all = 0;
6464
boolean hasNext=true;
6565
Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize());
6666
while (hasNext) {
67-
Page<Mention> allPersMentions = mentionService.getAll(pageRequest);
67+
Page<Mention> allPersMentions = mentionService.getAllWithoutPersistentUser(pageRequest);
6868
hasNext = allPersMentions.hasNext();
69-
long number = allPersMentions.getTotalElements();
7069
for (Mention onePersMentions : allPersMentions) {
7170
if (!onePersMentions.hasPersistentUser()) {
71+
lfdNr++;
72+
all++;
7273
String screenName = onePersMentions.getScreenName();
74+
log.debug("### mentionService.getAll from DB ("+lfdNr+"): "+screenName);
7375
screenNames.add(screenName);
7476
}
7577
}
7678
pageRequest = pageRequest.next();
7779
}
7880
int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls();
7981
for(String screenName:screenNames){
82+
lfdNr++;
83+
log.debug("### twitterApiService.getUserProfileForScreenName("+screenName+") from Twiiter API ("+lfdNr+" of "+all+")");
8084
TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName);
8185
if(userProfile!=null) {
8286
TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn, userProfile);
8387
userProfileList.add(userMsg);
8488
}
85-
log.debug("### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls);
89+
log.debug(msg+"### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls);
8690
try {
8791
Thread.sleep(millisToWaitBetweenTwoApiCalls);
8892
} catch (InterruptedException e) {

‎src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,22 +53,22 @@ public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiSer
5353

5454
@Override
5555
public List<TwitterProfileMessage> splitMessage(Message<TaskMessage> message) {
56-
String msg = "mqUpdateUserProfiles.splitMessage: ";
56+
String msg = "### mqUpdateUserProfiles.splitMessage: ";
5757
log.debug(msg+ " START");
5858
CountedEntities countedEntities = countedEntitiesService.countAll();
5959
TaskMessage msgIn = message.getPayload();
6060
long id = msgIn.getTaskId();
6161
Task task = taskService.findById(id);
6262
task = taskService.start(task,countedEntities);
63-
int allLoop = 0;
6463
int loopId = 0;
6564
boolean hasNext=true;
6665
List<Long> worklistProfileTwitterIds = new ArrayList<>();
6766
Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize());
6867
while (hasNext) {
6968
Page<User> userProfileTwitterIds = userService.getAll(pageRequest);
7069
for(User user:userProfileTwitterIds.getContent()){
71-
log.debug(msg+ " userService.getAllTwitterIds: "+user.getIdTwitter());
70+
loopId++;
71+
log.debug(msg+ "### userService.getAllTwitterIds: ("+loopId+") "+user.getIdTwitter());
7272
worklistProfileTwitterIds.add(user.getIdTwitter());
7373
}
7474
hasNext = userProfileTwitterIds.hasNext();
@@ -78,20 +78,20 @@ public List<TwitterProfileMessage> splitMessage(Message<TaskMessage> message) {
7878
int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls();
7979
List<TwitterProfileMessage> userProfileList = new ArrayList<>();
8080
for(Long userProfileTwitterId:worklistProfileTwitterIds){
81-
allLoop++;
82-
loopId++;
83-
String counter = " ( " + loopId + " from " + number + " ) [" + allLoop + "] ";
81+
String counter = " ( " + loopId + " from " + number + " ) ";
8482
log.debug(msg + counter);
8583
TwitterProfile userProfile = null;
8684
try {
85+
log.debug(msg+"### twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter);
8786
userProfile = twitterApiService.getUserProfileForTwitterId(userProfileTwitterId);
8887
} catch (RateLimitExceededException e) {
89-
log.error(msg + counter+ "twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") ",e);
88+
log.error(msg + "### ERROR: twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter,e);
9089
}
9190
if(userProfile != null){
9291
TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile);
9392
userProfileList.add(userMsg);
9493
}
94+
log.debug(msg + "### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls);
9595
try {
9696
Thread.sleep(millisToWaitBetweenTwoApiCalls);
9797
} catch (InterruptedException e) {

‎src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java‎

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,16 @@ public class TwitterApiServiceImpl implements TwitterApiService {
3030
public List<Tweet> findTweetsForSearchQuery() {
3131
String msg = MSG+"findTweetsForSearchQuery: ";
3232
log.debug(msg);
33-
List<Tweet> fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets();
33+
List<Tweet> fetchedTweets;
34+
try {
35+
fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets();
36+
} catch (Exception e) {
37+
fetchedTweets = new ArrayList<>();
38+
log.debug(msg + e.getMessage());
39+
e.printStackTrace();
40+
}
3441
msg += " result: ";
35-
if(fetchedTweets == null){
42+
if(fetchedTweets.size()==0){
3643
log.debug(msg+" result.size: 0");
3744
return new ArrayList<>();
3845
} else {
@@ -45,7 +52,14 @@ public List<Tweet> findTweetsForSearchQuery() {
4552
public Tweet findOneTweetById(long id) {
4653
String msg = MSG+"findOneTweetById: "+id;
4754
log.debug(msg);
48-
Tweet result = getTwitterProxy().timelineOperations().getStatus(id);
55+
Tweet result;
56+
try {
57+
result = getTwitterProxy().timelineOperations().getStatus(id);
58+
} catch (Exception e){
59+
result = null;
60+
log.debug(msg + e.getMessage());
61+
e.printStackTrace();
62+
}
4963
msg += " result: ";
5064
log.debug(msg+" Id: "+result.getId());
5165
return result;
@@ -55,7 +69,14 @@ public Tweet findOneTweetById(long id) {
5569
public List<TwitterProfile> getUserProfilesForTwitterIds(long... userProfileTwitterIds) {
5670
String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterIds;
5771
log.debug(msg);
58-
List<TwitterProfile> result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds);
72+
List<TwitterProfile> result;
73+
try {
74+
result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds);
75+
} catch (Exception e){
76+
result = null;
77+
log.debug(msg + e.getMessage());
78+
e.printStackTrace();
79+
}
5980
msg += " result: ";
6081
log.debug(msg+" size: "+result.size());
6182
return result;
@@ -65,7 +86,14 @@ public List<TwitterProfile> getUserProfilesForTwitterIds(long... userProfileTwit
6586
public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) {
6687
String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterId;
6788
log.debug(msg);
68-
TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId);
89+
TwitterProfile result;
90+
try {
91+
result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId);
92+
} catch (Exception e) {
93+
result = null;
94+
log.debug(msg + e.getMessage());
95+
e.printStackTrace();
96+
}
6997
msg += " result: ";
7098
log.debug(msg+" Id: "+result.getId());
7199
log.debug(msg+" ScreenName: "+result.getScreenName());
@@ -77,8 +105,14 @@ public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) {
77105
public TwitterProfile getUserProfileForScreenName(String screenName) {
78106
String msg = MSG+"getUserProfileForScreenName: "+screenName;
79107
log.debug(msg);
80-
TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(screenName);
81-
log.debug(msg+result.getId());
108+
TwitterProfile result;
109+
try {
110+
result= getTwitterProxy().userOperations().getUserProfile(screenName);
111+
} catch (Exception e) {
112+
result = null;
113+
log.debug(msg + e.getMessage());
114+
e.printStackTrace();
115+
}
82116
msg += " result: ";
83117
log.debug(msg+" ScreenName: "+result.getScreenName());
84118
log.debug(msg+" Name: "+result.getName());
@@ -89,7 +123,14 @@ public TwitterProfile getUserProfileForScreenName(String screenName) {
89123
public List<TwitterProfile> findUsersFromDefinedList(String screenName,String fetchUserListName) {
90124
String msg = MSG+"findUsersFromDefinedList: "+fetchUserListName+" ";
91125
log.debug(msg);
92-
List<TwitterProfile> result = getTwitterProxy().listOperations().getListMembers(screenName,fetchUserListName);
126+
List<TwitterProfile> result;
127+
try {
128+
result = getTwitterProxy().listOperations().getListMembers(screenName, fetchUserListName);
129+
} catch (Exception e) {
130+
result = new ArrayList<>();
131+
log.debug(msg + e.getMessage());
132+
e.printStackTrace();
133+
}
93134
log.debug(msg+" result.size: "+result.size());
94135
return result;
95136
}

0 commit comments

Comments
 (0)