diff -Naur a/res/res_rtp_asterisk.c b/res/res_rtp_asterisk.c
--- a/res/res_rtp_asterisk.c	2015-08-24 11:29:13.000000000 +0200
+++ b/res/res_rtp_asterisk.c	2015-08-25 10:12:35.291485943 +0200
@@ -3148,15 +3148,15 @@
 
 		/* Re-calculate last TS */
 		rtp->lastts = rtp->lastts + ms * rate;
-		if (ast_tvzero(frame->delivery)) {
-			/* If this isn't an absolute delivery time, Check if it is close to our prediction,
-			   and if so, go with our prediction */
-			if (abs((int)rtp->lastts - pred) < MAX_TIMESTAMP_SKEW) {
+		/* If this isn't an absolute delivery time, Check if it is close to our prediction,
+		   and if so, go with our prediction */
+		if (abs((int)rtp->lastts - pred) < MAX_TIMESTAMP_SKEW) {
+			if (ast_tvzero(frame->delivery)) {
 				rtp->lastts = pred;
-			} else {
-				ast_debug(3, "Difference is %d, ms is %u\n", abs((int)rtp->lastts - pred), ms);
-				mark = 1;
 			}
+		} else {
+			ast_debug(3, "Difference is %d, ms is %u\n", abs((int)rtp->lastts - pred), ms);
+			mark = 1;
 		}
 	} else if (frame->frametype == AST_FRAME_VIDEO) {
 		mark = frame->subclass.frame_ending;