Hello and welcome to our community! Is this your first visit?
Register
Enjoy an ad free experience by logging in. Not a member yet? Register.
Results 1 to 8 of 8
  1. #1
    Regular Coder
    Join Date
    Sep 2010
    Posts
    331
    Thanks
    9
    Thanked 6 Times in 6 Posts

    How to filter or avoid duplicate data?

    In a real-time message app, I need to avoid duplicate data. For example, I don't want "something" echoed twice unless it is in the database twice. How can I filter out any data that was already displayed? I haven't been able to find any uses for the lastEventId property.
    On page load:
    PHP Code:
    <div id="chatMsgs">';
                        $con = new PDO(DB_DSN, DB_USERNAME, DB_PASSWORD);    
                        $con->exec("SET CHARACTER SET utf8"); 
                        $getMsgs = "SELECT * FROM `messages` WHERE `to` = '
    $_SESSION[username]' OR `from` = 'S_SESSION[username]' ORDER BY `sent` ASC";
                        $receivedMsgs = $con->query($getMsgs); 
                        foreach ($receivedMsgs->fetchAll() as $msg) { 
                            $getLastID = "SELECT COUNT(id) FROM `messages`";
                            $LIQuery = $con->query($getLastID);
                            $lastID = $LIQuery->fetchColumn();
                            $getUI = "SELECT * FROM `users` WHERE `username` = '
    $msg[from]'";
                            $uiQuery = $con->query($getUI);
                            while ($UI = $uiQuery->fetch(PDO::FETCH_ASSOC)) {
                                echo "<article style='
    border-bottom:1px solid #FFFFFF;padding:5px;margin-bottom:2%;'>
                                           
    <img style='float:left;margin:0px 3px 5px 0px;width:50px;height:50px;' src='{$UI['profile_pic']}' alt='{$UI['full_name']}' />
                                           <
    p style='float:right;'>". Agotime($msg[sent]) ."</p>
                                           <
    a style='position:absolute;' href='/profile?uid={$UI[id]}'>{$UI['full_name']}</a>
                                           <
    br />
                                           <
    h4>@{$msg['from']}</h4>
                                           <
    p>{$msg['message']}</p>
                                           </
    article>";
                                                          
                            }
                        }
                        echo '
                        </div> 
    Getting the messages:
    Code:
    function getMsgs() {
    	if (typeof(EventSource) !== "undefined") {
    		var source = new EventSource('/ajax/gMessages.php');
    		var last_id;
    		source.addEventListener('message', function(event) {
    			var now_id = event.lastEventId;
    			if (last_id != now_id && document.getElementById('chatMsgs').innerHTML != event.data) {
    				last_id = event.lastEventId;
    				document.getElementById('chatMsgs').innerHTML += event.data;
    			}
    		}, false);
    	} else { 
    		var LID = document.getElementsByClassName('LII')[0].innerHTML;
    		var hr = new XMLHttpRequest();
    		var url = "/ajax/gMsgs.php?lid="+LID;
    		hr.open("GET", url, true);
    		hr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
    		hr.onreadystatechange = function() {
    			if (hr.readyState === 4 && hr.status === 200) {
    				var return_data = hr.responseText;
    				document.getElementById('chatMsgs').innerHTML += return_data;
    			}
    		}
    		hr.send();
    	}
    }
    if (typeof(EventSource) == "undefined") {
    	document.getElementById('openChat').addEventListener('mousedown', function() {setInterval(getMsgs, 3000);}, false);
    } else {
    	document.getElementById('openChat').addEventListener('mousedown', getMsgs, false);
    }
    Getting messages from the database:
    PHP Code:
    <?php
    session_start
    ();
    header('Content-Type: text/event-stream');
    header('Cache-Control: no-cache');
    include 
    "config.php"
    include 
    "functions.php";
    $con = new PDO(DB_DSNDB_USERNAMEDB_PASSWORD);    
    $con->exec("SET CHARACTER SET utf8"); 
    $getMsgs "SELECT * FROM `messages` WHERE `to` = '$_SESSION[username]' OR `from` = 'S_SESSION[username]' ORDER BY `sent` ASC";
    $receivedMsgs $con->query($getMsgs); 
    foreach (
    $receivedMsgs->fetchAll() as $msg) { 
        
    $getUI "SELECT * FROM `users` WHERE `username` = '$msg[from]'";
        
    $uiQuery $con->query($getUI);
        while (
    $UI $uiQuery->fetch(PDO::FETCH_ASSOC)) {
            echo 
    "id: {$msg['id']}\n";
            echo 
    "data: <article style='border-bottom:1px solid #FFFFFF;padding:5px;margin-bottom:2%;'>\n";
            echo 
    "data: <img style='float:left;margin:0px 3px 5px 0px;width:50px;height:50px;' src='{$UI[profile_pic]}' alt='{$UI[full_name]}' />\n";
            echo 
    "data: <p style='float:right;'>"Agotime($msg[sent]) ."</p>\n";
            echo 
    "data: <a style='position:absolute;' href='/profile?uid={$UI[id]}'>{$UI['full_name']}</a><br />\n";
            echo 
    "data: <h4>@{$msg['from']}</h4>\n";
            echo 
    "data: <p>{$msg['message']}</p>\n";
            echo 
    "data: </article>\n\n";
        }
            
    ob_flush();
            
    flush();
            
    sleep(1);
    }
    ?>
    Coding is a challenge, get used to it
    Always remember to debug
    Try the guess & check method
    Break it down into simple steps

  • #2
    Senior Coder rnd me's Avatar
    Join Date
    Jun 2007
    Location
    Urbana
    Posts
    4,184
    Thanks
    10
    Thanked 569 Times in 550 Posts
    Code:
    var used={}
    
    		source.addEventListener('message', function(event) {
    			var now_id = event.lastEventId;
    			if (last_id != now_id && document.getElementById('chatMsgs').innerHTML != event.data) {
    				last_id = event.lastEventId;
    
          if(!used[event.data]){
    				document.getElementById('chatMsgs').innerHTML += event.data;
                                    used[event.data]=1;
          }
    
    
    			}
    		}, false);
    my site (updated 13/9/26)
    BROWSER STATS [% share] (2014/1/19) IE7:0.2, IE8:6.7, IE11:7.4, IE9:3.8, IE10:4.4, FF:18.3, CH:43.6, SF:7.8, MOBILE:27.5

  • #3
    Regular Coder
    Join Date
    Sep 2010
    Posts
    331
    Thanks
    9
    Thanked 6 Times in 6 Posts
    The same problem continues. The chatbox just continuously repeats the same data in an endless loop.
    Code:
    function getMsgs() {
    	if (typeof(EventSource) !== "undefined") {
    		var source = new EventSource('/ajax/gMessages.php');
    		var last_id;
    		var used = {};
    		source.addEventListener('message', function(event) {
    			var now_id = event.lastEventId;
    			if (last_id != now_id && document.getElementById('chatMsgs').innerHTML != event.data) {
    				last_id = event.lastEventId;
    
          if (!used[event.data]) {
    				document.getElementById('chatMsgs').innerHTML += event.data;
                                    used[event.data] = 1;
          }
    			}
    		}, false);
    	} else { 
    		var LID = document.getElementsByClassName('LII')[0].innerHTML;
    		var hr = new XMLHttpRequest();
    		var url = "/ajax/gMsgs.php?lid="+LID;
    		hr.open("GET", url, true);
    		hr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
    		hr.onreadystatechange = function() {
    			if (hr.readyState === 4 && hr.status === 200) {
    				var return_data = hr.responseText;
    				document.getElementById('chatMsgs').innerHTML += return_data;
    			}
    		}
    		hr.send();
    	}
    }
    Coding is a challenge, get used to it
    Always remember to debug
    Try the guess & check method
    Break it down into simple steps

  • #4
    Senior Coder rnd me's Avatar
    Join Date
    Jun 2007
    Location
    Urbana
    Posts
    4,184
    Thanks
    10
    Thanked 569 Times in 550 Posts
    Quote Originally Posted by elitis View Post
    The same problem continues. The chatbox just continuously repeats the same data in an endless loop.


    if it does so after my alteration is applied, then you need to adjust the server to set event.data the same each time a repetitive message comes across.

    looks like your agotime piece changes with each call and thus prevents collision detection since it's not really a collision, it's unique.

    i don't see the point of agotime, you can track that on the client.

    make the same input equal the same output and my fix will handle the rest.
    my site (updated 13/9/26)
    BROWSER STATS [% share] (2014/1/19) IE7:0.2, IE8:6.7, IE11:7.4, IE9:3.8, IE10:4.4, FF:18.3, CH:43.6, SF:7.8, MOBILE:27.5

  • #5
    Regular Coder
    Join Date
    Sep 2010
    Posts
    331
    Thanks
    9
    Thanked 6 Times in 6 Posts
    Quote Originally Posted by rnd me View Post
    if it does so after my alteration is applied, then you need to adjust the server to set event.data the same each time a repetitive message comes across.

    looks like your agotime piece changes with each call and thus prevents collision detection since it's not really a collision, it's unique.

    i don't see the point of agotime, you can track that on the client.

    make the same input equal the same output and my fix will handle the rest.
    All agotime does is find how long ago something was posted, could that really affect other code? How exactly would I adjust the server? I believe I'm running Apache, and what exactly do you mean by "make the same input..."? Could you go a little more in depth on this?
    Coding is a challenge, get used to it
    Always remember to debug
    Try the guess & check method
    Break it down into simple steps

  • #6
    Senior Coder rnd me's Avatar
    Join Date
    Jun 2007
    Location
    Urbana
    Posts
    4,184
    Thanks
    10
    Thanked 569 Times in 550 Posts
    normally i don't touch php, but what the hey, i'll try:

    update yours with this:
    Code:
    while ($UI = $uiQuery->fetch(PDO::FETCH_ASSOC)) { 
            echo "id: {$msg['id']}\n"; 
            echo "data: <article style='border-bottom:1px solid #FFFFFF;padding:5px;margin-bottom:2%;'>\n"; 
            echo "data: <img style='float:left;margin:0px 3px 5px 0px;width:50px;height:50px;' src='{$UI[profile_pic]}' alt='{$UI[full_name]}' />\n"; 
            echo "data: <p style='float:right;'> N/A  </p>\n"; 
            echo "data: <a style='position:absolute;' href='/profile?uid={$UI[id]}'>{$UI['full_name']}</a><br />\n"; 
            echo "data: <h4>@{$msg['from']}</h4>\n"; 
            echo "data: <p>{$msg['message']}</p>\n"; 
            echo "data: </article>\n\n"; 
        }
    now, as i see it, asking for the same msg id should give back the same response each time; i don't see any other dates or unique transaction IDs to break the cache...
    my site (updated 13/9/26)
    BROWSER STATS [% share] (2014/1/19) IE7:0.2, IE8:6.7, IE11:7.4, IE9:3.8, IE10:4.4, FF:18.3, CH:43.6, SF:7.8, MOBILE:27.5

  • #7
    Regular Coder
    Join Date
    Sep 2010
    Posts
    331
    Thanks
    9
    Thanked 6 Times in 6 Posts
    Well then other than the initial messages pre-loaded into the chatbox there seems to be no more duplicates. Why is it that this works when that page is set so the browser won't cache it?
    Coding is a challenge, get used to it
    Always remember to debug
    Try the guess & check method
    Break it down into simple steps

  • #8
    Senior Coder rnd me's Avatar
    Join Date
    Jun 2007
    Location
    Urbana
    Posts
    4,184
    Thanks
    10
    Thanked 569 Times in 550 Posts
    Quote Originally Posted by elitis View Post
    Well then other than the initial messages pre-loaded into the chatbox there seems to be no more duplicates. Why is it that this works when that page is set so the browser won't cache it?
    eventsource doesn't cache.
    my site (updated 13/9/26)
    BROWSER STATS [% share] (2014/1/19) IE7:0.2, IE8:6.7, IE11:7.4, IE9:3.8, IE10:4.4, FF:18.3, CH:43.6, SF:7.8, MOBILE:27.5


  •  

    Posting Permissions

    • You may not post new threads
    • You may not post replies
    • You may not post attachments
    • You may not edit your posts
    •